diff --git a/CHANGELOG.md b/CHANGELOG.md index c357cab65..dda8c7eb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,8 @@ [#1896](https://github.com/nextcloud/cookbook/pull/1896) @seyfeb - Make API interface cleaner by only using string identifiers for recipes [#1909](https://github.com/nextcloud/cookbook/pull/1909) @christianlupus +- Add filter for timestamps to output canonical ISO8601 timestamps in the form of YYYY-MM-DDTHH:mm:ss±hh:mm See issue [#1543](https://github.com/nextcloud/cookbook/issues/1543) + [#1903](https://github.com/nextcloud/cookbook/pull/1903) @seyfeb ### Maintenance - Preparation for migration to vue.js 3 diff --git a/lib/Exception/InvalidTimestampException.php b/lib/Exception/InvalidTimestampException.php new file mode 100644 index 000000000..78cf1e596 --- /dev/null +++ b/lib/Exception/InvalidTimestampException.php @@ -0,0 +1,9 @@ +l = $l; + $this->logger = $logger; + $this->timestampHelper = $tsHelper; + } + + public function apply(array &$json): bool { + $changed = false; + + $this->fixTimestamp($json, 'dateCreated', $changed); + $this->fixTimestamp($json, 'dateModified', $changed); + $this->fixTimestamp($json, 'datePublished', $changed); + + return $changed; + } + + private function fixTimestamp(array &$json, string $type, bool &$changed): void { + if (!isset($json[$type])) { + $json[$type] = null; + $changed = true; + return; + } + + $orig = $json[$type]; + try { + $json[$type] = $this->timestampHelper->parseTimestamp($json[$type]); + } catch (InvalidTimestampException $ex) { + $json[$type] = null; + } + + if ($orig !== $json[$type]) { + $changed = true; + } + } +} diff --git a/lib/Helper/Filter/JSON/JSONFilter.php b/lib/Helper/Filter/JSON/JSONFilter.php index b10eca20b..7724010bd 100644 --- a/lib/Helper/Filter/JSON/JSONFilter.php +++ b/lib/Helper/Filter/JSON/JSONFilter.php @@ -15,6 +15,7 @@ public function __construct( CleanCategoryFilter $cleanCategoryFilter, FixRecipeYieldFilter $fixRecipeYieldFilter, FixKeywordsFilter $fixKeywordsFilter, + FixTimestampsFilter $fixTimestampsFilter, FixToolsFilter $fixToolsFilter, FixIngredientsFilter $fixIngredientsFilter, FixInstructionsFilter $fixInstructionsFilter, @@ -32,6 +33,7 @@ public function __construct( $cleanCategoryFilter, $fixRecipeYieldFilter, $fixKeywordsFilter, + $fixTimestampsFilter, $fixToolsFilter, $fixIngredientsFilter, $fixInstructionsFilter, diff --git a/lib/Helper/TimestampHelper.php b/lib/Helper/TimestampHelper.php new file mode 100644 index 000000000..4f19a2287 --- /dev/null +++ b/lib/Helper/TimestampHelper.php @@ -0,0 +1,165 @@ +l = $l; + } + + /** + * Parse a string representation of a timestamp and format according to ISO8601. + * For reference, see Wikipedia. + * + * The output time span is in the form `YYYY-MM-DDThh:mm:ss±hh:mm`, e.g., `2023-11-25T14:25:36+01:00`. + * - YYYY - year + * - MM - month + * - DD - day + * - hh - hours + * - mm - minutes + * - ss - seconds + * + * The time before the ± defines the local time, the time after ± defines the timezone of the timestamp. In order to + * calculate the time in UTC, the value after ± needs to be added/subtracted from the local time. + * + * @param string $timestamp The timestamp to parse + * @return string The timestamp in ISO8601 format + * @throws InvalidTimestampException if the input data could not be parsed successfully. + */ + public function parseTimestamp(string $timestamp): string { + // For now, we only support the ISO8601 format because it is required in the schema.org standard + try { + return $this->parseIsoFormat($timestamp); + } catch (InvalidTimestampException) { + // We do nothing here. Check the next format + } + + // No more formats are available. + throw new InvalidTimestampException($this->l->t('Could not parse timestamp {timestamp}', ['timestamp' => $timestamp])); + } + + /** + * Parses the string $timestamp and checks if it has a valid ISO 8601 format. Otherwise, throws + * InvalidTimestampException. + * + * For reference of ISO 8601, see Wikipedia. + * @param string $timestamp Timestamp to be parsed. + * @return string + * @throws InvalidTimestampException if $timestamp does not comply to ISO 8601. + */ + private function parseIsoFormat(string $timestamp): string { + try { + return $this->parseIsoCalendarDateFormat($timestamp, '-'); + } catch (InvalidTimestampException) { // Check next format + } + try { + return $this->parseIsoCalendarDateFormat($timestamp, ''); + } catch (InvalidTimestampException) { // Check next format + } + try { + return $this->parseIsoWeekDateFormat($timestamp, '-'); + } catch (InvalidTimestampException) { // Check next format + } + + return $this->parseIsoWeekDateFormat($timestamp, ''); + } + + /** + * Parses the string $timestamp and checks if it has a valid ISO 8601 with date as year, month, and day. + * Otherwise, throws InvalidTimestampException. + * + * For reference of ISO 8601, see Wikipedia. + * @param string $timestamp Timestamp to be parsed + * @param string $dateSeparator Separator to be used between `YYYY`, `mm`, and `dd`. + * @return string + * @throws InvalidTimestampException if $timestamp does not comply to ISO 8601 with week and weekday. + */ + private function parseIsoCalendarDateFormat(string $timestamp, string $dateSeparator = '-'): string { + $date = "Y".$dateSeparator."m".$dateSeparator."d"; + + return $this->parseIsoTimestampWithTimeFormats($timestamp, $date); + } + + /** + * Parses the string $timestamp and checks if it has a valid ISO 8601 format with date defined as week and weekday. + * Otherwise, throws InvalidTimestampException. + * + * For reference of ISO 8601, see Wikipedia. + * @param string $timestamp Timestamp to be parsed + * @param string $dateSeparator Separator to be used between `YYYY`, `mm`, and `dd`. + * @return string + * @throws InvalidTimestampException if $timestamp does not comply to ISO 8601 with week and weekday. + */ + private function parseIsoWeekDateFormat(string $timestamp, string $dateSeparator = '-'): string { + $pattern = "/^(?!$)(\d\d\d\d)" . $dateSeparator . "W(\d\d)" . $dateSeparator . "(\d)T.*$/"; + $ret = preg_match($pattern, trim($timestamp), $matches); + + if ($ret === 1) { + // Convert week format to calendar format for date + $tmpDate = new DateTime('midnight'); + // $matches[0] is the complete string + // $matches[1] to $matches[3] is year, week, weekday + $tmpDate->setISODate((int)$matches[1], (int)$matches[2], (int)$matches[3]); + $tmpDateString = $tmpDate->format('Y-m-d\TH:i:sP'); + + // Combine converted date with original time + $timePartOfTimestamp = substr($timestamp, 8 + 2 * mb_strlen($dateSeparator)); + $datePartOfConvertedTimestamp = substr($tmpDateString, 0, 10); + $updatedTimestamp = $datePartOfConvertedTimestamp . $timePartOfTimestamp; + + // Parse complete date including time + $dateFormat = 'Y-m-d'; + return $this->parseIsoTimestampWithTimeFormats($updatedTimestamp, $dateFormat); + } + + throw new InvalidTimestampException($this->l->t('Could not parse timestamp {timestamp}', ['timestamp' => $timestamp])); + } + + + /** + * Parses the string $timestamp and checks if it has a valid ISO 8601. Uses the date format given by $dateFormat + * and checks several allowed formats for the time. + * + * For reference of ISO 8601, see Wikipedia. + * @param string $timestamp Timestamp to be parsed + * @param string $dateFormat Format to be used for the date portion + * @return string + * @throws InvalidTimestampException if $timestamp does not comply to any of the checked formats allowed by ISO 8601. + */ + private function parseIsoTimestampWithTimeFormats(string $timestamp, string $dateFormat): string { + // Try parsing timestamp without milliseconds + $dt = DateTimeImmutable::createFromFormat($dateFormat . "\\TH:i:sP", $timestamp); + if($dt) { + return $dt->format(self::OUTPUT_FORMAT); + } + + // Try parsing timestamp with dot-separated milliseconds + $dt = DateTimeImmutable::createFromFormat($dateFormat . "\\TH:i:s.vP", $timestamp); + if($dt) { + return $dt->format(self::OUTPUT_FORMAT); + } + + // Try parsing timestamp with comma-separated milliseconds + $dt = DateTimeImmutable::createFromFormat($dateFormat . "\\TH:i:s,vP", $timestamp); + if($dt) { + return $dt->format(self::OUTPUT_FORMAT); + } + throw new InvalidTimestampException($this->l->t('Could not parse timestamp {timestamp}', ['timestamp' => $timestamp])); + } + +} diff --git a/tests/Unit/Helper/Filter/JSON/FixTimestampsFilterTest.php b/tests/Unit/Helper/Filter/JSON/FixTimestampsFilterTest.php new file mode 100644 index 000000000..02e7b6a15 --- /dev/null +++ b/tests/Unit/Helper/Filter/JSON/FixTimestampsFilterTest.php @@ -0,0 +1,151 @@ +createStub(IL10N::class); + $l->method('t')->willReturnArgument(0); + $logger = $this->createStub(LoggerInterface::class); + + $this->timestampHelper = $this->createMock(TimestampHelper::class); + + $this->dut = new FixTimestampsFilter($l, $logger, $this->timestampHelper); + + $this->stub = [ + 'name' => 'The name of the recipe', + 'id' => 1234, + ]; + } + + public function dpNonExisting() { + return [ + [false, false, false], + [true, false, false], + [false, true, false], + [false, false, true], + ]; + } + + /** @dataProvider dpNonExisting */ + public function testNonExisting($preExists, $cookExists, $totalExists) { + if ($preExists) { + $this->stub['dateCreated'] = '2000-01-01T01:01:00+00:00'; + } + if ($cookExists) { + $this->stub['dateModified'] = '2001-01-01T01:01:00+01:00'; + } + if ($totalExists) { + $this->stub['datePublished'] = '2002-01-01T01:01:00,123-01:30'; + } + + $recipe = $this->stub; + $this->timestampHelper->method('parseTimestamp')->willReturnArgument(0); + + $this->assertTrue($this->dut->apply($recipe)); + + $this->stub['dateCreated'] ??= null; + $this->stub['dateModified'] ??= null; + $this->stub['datePublished'] ??= null; + + $this->assertEquals($this->stub, $recipe); + } + + public function dpSuccess() { + return [ + ['2000-01-01T01:01:00+00:00', '2001-01-01T01:01:00+01:00', '2002-01-01T01:01:00-01:30', + '2000-01-01T01:01:00+00:00', '2001-01-01T01:01:00+01:00', '2002-01-01T01:01:00-01:30', false], + + ['2000-01-01T01:01:00Z', '2001-01-01T01:01:00+01:00', '2002-01-01T01:01:00,123-01:30', + '2000-01-01T01:01:00+00:00', '2001-01-01T01:01:00+01:00', '2002-01-01T01:01:00-01:30', true], + ['2001-01-01T01:01:00+01:00', '2000-01-01T01:01:00Z', '2002-01-01T01:01:00,123-01:30', + '2001-01-01T01:01:00+01:00', '2000-01-01T01:01:00+00:00', '2002-01-01T01:01:00-01:30', true], + ['2001-01-01T01:01:00+01:00', '2002-01-01T01:01:00,123-01:30', '2000-01-01T01:01:00Z', + '2001-01-01T01:01:00+01:00', '2002-01-01T01:01:00-01:30', '2000-01-01T01:01:00+00:00', true], + ]; + } + + /** @dataProvider dpSuccess */ + public function testSuccess( + $originalCreated, $originalModified, $originalPublished, + $expectedCreated, $expectedModified, $expectedPublished, + $expectedChange + ) { + $this->timestampHelper->method('parseTimestamp')->willReturnMap([ + ['2000-01-01T01:01:00+00:00', '2000-01-01T01:01:00+00:00'], + ['2001-01-01T01:01:00+01:00', '2001-01-01T01:01:00+01:00'], + ['2002-01-01T01:01:00-01:30', '2002-01-01T01:01:00-01:30'], + ['2002-01-01T01:01:00,123-01:30', '2002-01-01T01:01:00-01:30'], + ['2000-01-01T01:01:00Z', '2000-01-01T01:01:00+00:00'], + ]); + + $this->stub['dateCreated'] = $originalCreated; + $this->stub['dateModified'] = $originalModified; + $this->stub['datePublished'] = $originalPublished; + + $recipe = $this->stub; + + $this->assertEquals($expectedChange, $this->dut->apply($recipe)); + + $this->stub['dateCreated'] = $expectedCreated; + $this->stub['dateModified'] = $expectedModified; + $this->stub['datePublished'] = $expectedPublished; + + $this->assertEquals($this->stub, $recipe); + } + + public function dpExceptions() { + return [ + ['invalid', '2000-01-02T01:01:00Z', '2000-01-03T01:01:00Z', null, '2000-01-02T01:01:00Z', '2000-01-03T01:01:00Z'], + ['2000-01-01T01:01:00Z', 'invalid', '2000-01-03T01:01:00Z', '2000-01-01T01:01:00Z', null, '2000-01-03T01:01:00Z'], + ['2000-01-01T01:01:00Z', '2000-01-02T01:01:00Z', 'invalid', '2000-01-01T01:01:00Z', '2000-01-02T01:01:00Z', null], + ]; + } + + /** @dataProvider dpExceptions */ + public function testExceptions( + $dateCreated, $dateModified, $datePublished, + $expectedCreated, $expectedModified, $expectedPublished + ) { + $this->timestampHelper->method('parseTimestamp')->willReturnCallback(function ($x) { + if ($x === 'invalid') { + throw new InvalidTimestampException(); + } else { + return $x; + } + }); + + $this->stub['dateCreated'] = $dateCreated; + $this->stub['dateModified'] = $dateModified; + $this->stub['datePublished'] = $datePublished; + + $recipe = $this->stub; + + $this->assertTrue($this->dut->apply($recipe)); + + $this->stub['dateCreated'] = $expectedCreated; + $this->stub['dateModified'] = $expectedModified; + $this->stub['datePublished'] = $expectedPublished; + + $this->assertEquals($this->stub, $recipe); + } +} diff --git a/tests/Unit/Helper/Filter/JSONFilterTest.php b/tests/Unit/Helper/Filter/JSONFilterTest.php index 653cc8d48..bbb106db3 100644 --- a/tests/Unit/Helper/Filter/JSONFilterTest.php +++ b/tests/Unit/Helper/Filter/JSONFilterTest.php @@ -12,6 +12,7 @@ use OCA\Cookbook\Helper\Filter\JSON\FixKeywordsFilter; use OCA\Cookbook\Helper\Filter\JSON\FixNutritionFilter; use OCA\Cookbook\Helper\Filter\JSON\FixRecipeYieldFilter; +use OCA\Cookbook\Helper\Filter\JSON\FixTimestampsFilter; use OCA\Cookbook\Helper\Filter\JSON\FixToolsFilter; use OCA\Cookbook\Helper\Filter\JSON\FixUrlFilter; use OCA\Cookbook\Helper\Filter\JSON\JSONFilter; @@ -32,6 +33,7 @@ class JSONFilterTest extends TestCase { private $cleanCategoryFilter; private $fixRecipeYieldFilter; private $fixKeywordsFilter; + private $fixTimestampsFilter; private $fixToolsFilter; private $fixIngredientsFilter; private $fixInstructionsFilter; @@ -50,6 +52,7 @@ protected function setUp(): void { $this->fixRecipeYieldFilter = $this->createStub(FixRecipeYieldFilter::class); $this->fixKeywordsFilter = $this->createStub(FixKeywordsFilter::class); $this->fixToolsFilter = $this->createStub(FixToolsFilter::class); + $this->fixTimestampsFilter = $this->createStub(FixTimestampsFilter::class); $this->fixIngredientsFilter = $this->createStub(FixIngredientsFilter::class); $this->fixInstructionsFilter = $this->createStub(FixInstructionsFilter::class); $this->fixDescriptionFilter = $this->createStub(FixDescriptionFilter::class); @@ -67,6 +70,7 @@ protected function setUp(): void { $this->cleanCategoryFilter, $this->fixRecipeYieldFilter, $this->fixKeywordsFilter, + $this->fixTimestampsFilter, $this->fixToolsFilter, $this->fixIngredientsFilter, $this->fixInstructionsFilter, @@ -98,6 +102,7 @@ public function testSequence() { $this->cleanCategoryFilter->method('apply')->willReturnCallback($closure()); $this->fixRecipeYieldFilter->method('apply')->willReturnCallback($closure()); $this->fixKeywordsFilter->method('apply')->willReturnCallback($closure()); + $this->fixTimestampsFilter->method('apply')->willReturnCallback($closure()); $this->fixToolsFilter->method('apply')->willReturnCallback($closure()); $this->fixIngredientsFilter->method('apply')->willReturnCallback($closure()); $this->fixInstructionsFilter->method('apply')->willReturnCallback($closure()); diff --git a/tests/Unit/Helper/TimestampHelperTest.php b/tests/Unit/Helper/TimestampHelperTest.php new file mode 100644 index 000000000..2dfd8532c --- /dev/null +++ b/tests/Unit/Helper/TimestampHelperTest.php @@ -0,0 +1,76 @@ +createStub(IL10N::class); + $l->method('t')->willReturnArgument(0); + + $this->dut = new TimestampHelper($l); + } + + public function dpIso() { + return [ + ['2000-01-01T01:01:00+00:00', '2000-01-01T01:01:00+00:00'], + ['2000-01-01T01:01:00Z', '2000-01-01T01:01:00+00:00'], + ['2000-01-01T01:01:00+05:30', '2000-01-01T01:01:00+05:30'], + ['2000-01-01T01:01:00-05:30', '2000-01-01T01:01:00-05:30'], + ['2000-01-01T01:01:00.5+00:00', '2000-01-01T01:01:00+00:00'], + ['2000-01-01T01:01:00,12+00:00', '2000-01-01T01:01:00+00:00'], + ['2000-01-01T01:01:00.123+00:00', '2000-01-01T01:01:00+00:00'], + ['2000-01-01T01:01:00.123Z', '2000-01-01T01:01:00+00:00'], + ['2000-01-01T01:01:00.123+01:00', '2000-01-01T01:01:00+01:00'], + ['2000-01-01T01:01:00.123-01:00', '2000-01-01T01:01:00-01:00'], + + + ['20000101T01:01:00.123-01:00', '2000-01-01T01:01:00-01:00'], + ['20000101T01:01:00-01:00', '2000-01-01T01:01:00-01:00'], + + ['2014-W01-2T01:01:00.123-01:00', '2013-12-31T01:01:00-01:00'], + ['2014-W01-2T01:01:00,123-01:00', '2013-12-31T01:01:00-01:00'], + ['2014-W01-2T01:01:00-01:00', '2013-12-31T01:01:00-01:00'], + + ['2014W012T01:01:00.123-01:00', '2013-12-31T01:01:00-01:00'], + ['2014W012T01:01:00,123-01:00', '2013-12-31T01:01:00-01:00'], + ['2014W012T01:01:00-01:00', '2013-12-31T01:01:00-01:00'], + ]; + } + + /** @dataProvider dpIso */ + public function testIsoFormat($input, $expectedOutput) { + $this->assertEquals($expectedOutput, $this->dut->parseTimestamp($input)); + } + + public function dpFail() { + return [ + ['Just some text'], + [''], + [' '], + ['123:45:46'], // Missing date + ['T12:10:00'], // Missing date + ['2000-01-01'], // Missing time + ['2000-01-01 01:01:00+00:00'], // Missing separator T + ['2000-01-01T01:01:00'], // Missing timezone + ['2000-01-01 T 01:01:00+00:00'], // There should be no spaces + ['01.11.2000T01:01:00+00:00'], // Wrong date + ['2000/12/01T01:01:00+00:00'], // Wrong date + ]; + } + + /** @dataProvider dpFail */ + public function testFailedFormat($input) { + $this->expectException(InvalidTimestampException::class); + $this->dut->parseTimestamp($input); + } +} diff --git a/tests/psalm-baseline.xml b/tests/psalm-baseline.xml index b08fd8e5e..755e475b4 100644 --- a/tests/psalm-baseline.xml +++ b/tests/psalm-baseline.xml @@ -1,5 +1,5 @@ - + getMessage()]]> @@ -51,11 +51,6 @@ setJSONValue($json, '@type', 'Recipe')]]> - - - apply($json, $recipeFile)]]> - - getAttribute @@ -94,11 +89,6 @@ IRootFolder - - - Provider - - GenericFileException