From aeaa820e47e89fcca48f03c11d8ae053b711f36c Mon Sep 17 00:00:00 2001 From: Sebastian Kurfuerst Date: Wed, 16 Jul 2025 18:58:25 +0200 Subject: [PATCH 01/10] FEATURE: support for different output formats, with differing documentRenderers and custom enumerators --- .../Domain/Dto/EnumeratedNode.php | 23 ++++- Classes/NodeEnumeration/NodeEnumerator.php | 31 ++++-- .../DocumentEnumeratorInterface.php | 30 ++++++ .../DocumentEnumerators/DefaultEnumerator.php | 21 ++++ .../DocumentEnumerators/LimitEnumerator.php | 33 +++++++ .../DocumentRendererInterface.php | 25 +++++ .../DocumentRenderers/FusionHtmlRenderer.php | 40 ++++++++ .../NodeRenderingExtensionManager.php | 95 +++++++++++++++---- .../NodeRendering/NodeRenderOrchestrator.php | 12 +-- Classes/NodeRendering/NodeRenderer.php | 15 ++- Configuration/Settings.yaml | 31 ++++-- 11 files changed, 298 insertions(+), 58 deletions(-) create mode 100644 Classes/NodeRendering/Extensibility/DocumentEnumeratorInterface.php create mode 100644 Classes/NodeRendering/Extensibility/DocumentEnumerators/DefaultEnumerator.php create mode 100644 Classes/NodeRendering/Extensibility/DocumentEnumerators/LimitEnumerator.php create mode 100644 Classes/NodeRendering/Extensibility/DocumentRendererInterface.php create mode 100644 Classes/NodeRendering/Extensibility/DocumentRenderers/FusionHtmlRenderer.php diff --git a/Classes/NodeEnumeration/Domain/Dto/EnumeratedNode.php b/Classes/NodeEnumeration/Domain/Dto/EnumeratedNode.php index 880459d..508a58e 100644 --- a/Classes/NodeEnumeration/Domain/Dto/EnumeratedNode.php +++ b/Classes/NodeEnumeration/Domain/Dto/EnumeratedNode.php @@ -41,17 +41,24 @@ final class EnumeratedNode implements \JsonSerializable */ protected $nodeTypeName; - private function __construct(string $contextPath, string $nodeIdentifier, string $nodeTypeName, array $arguments) + /** + * The renderer implementation to use for this EnumeratedNode; + * a key from Settings at Flowpack.DecoupledContentStore.extensions.documentRenderers.[key] + */ + public readonly string $rendererId; + + private function __construct(string $contextPath, string $nodeIdentifier, string $nodeTypeName, array $arguments, string $rendererId = '') { $this->contextPath = $contextPath; $this->nodeIdentifier = $nodeIdentifier; $this->nodeTypeName = $nodeTypeName; $this->arguments = $arguments; + $this->rendererId = $rendererId; } static public function fromNode(NodeInterface $node, array $arguments = []): self { - return new self($node->getContextPath(), $node->getIdentifier(), $node->getNodeType()->getName(), $arguments); + return new self($node->getContextPath(), $node->getIdentifier(), $node->getNodeType()->getName(), $arguments, ''); } static public function fromJsonString(string $enumeratedNodeString): self @@ -60,7 +67,7 @@ static public function fromJsonString(string $enumeratedNodeString): self if (!is_array($tmp)) { throw new \Exception('EnumeratedNode cannot be constructed from: ' . $enumeratedNodeString); } - return new self($tmp['contextPath'], $tmp['nodeIdentifier'], $tmp['nodeTypeName'] ?? '', $tmp['arguments']); + return new self($tmp['contextPath'], $tmp['nodeIdentifier'], $tmp['nodeTypeName'] ?? '', $tmp['arguments'], $tmp['rendererId']); } public function jsonSerialize() @@ -69,7 +76,8 @@ public function jsonSerialize() 'contextPath' => $this->contextPath, 'nodeIdentifier' => $this->nodeIdentifier, 'nodeTypeName' => $this->nodeTypeName, - 'arguments' => $this->arguments + 'arguments' => $this->arguments, + 'rendererId' => $this->rendererId, ]; } @@ -105,6 +113,11 @@ public function getArguments(): array public function debugString(): string { - return sprintf('%s %s %s(%s)', $this->nodeTypeName, $this->nodeIdentifier, $this->arguments ? http_build_query($this->arguments) . ' ' : '', $this->contextPath); + return sprintf('%s %s %s(%s) - %s', $this->nodeTypeName, $this->nodeIdentifier, $this->arguments ? http_build_query($this->arguments) . ' ' : '', $this->contextPath, $this->rendererId); + } + + public function withRendererId(string $rendererId): self + { + return new self($this->contextPath, $this->nodeIdentifier, $this->nodeTypeName, $this->arguments, $rendererId); } } diff --git a/Classes/NodeEnumeration/NodeEnumerator.php b/Classes/NodeEnumeration/NodeEnumerator.php index 41d7646..a29e361 100644 --- a/Classes/NodeEnumeration/NodeEnumerator.php +++ b/Classes/NodeEnumeration/NodeEnumerator.php @@ -11,6 +11,7 @@ use Flowpack\DecoupledContentStore\NodeEnumeration\Domain\Repository\RedisEnumerationRepository; use Flowpack\DecoupledContentStore\NodeEnumeration\Domain\Service\NodeContextCombinator; use Flowpack\DecoupledContentStore\NodeRendering\Dto\NodeRenderingCompletionStatus; +use Flowpack\DecoupledContentStore\NodeRendering\Extensibility\NodeRenderingExtensionManager; use Flowpack\DecoupledContentStore\PrepareContentRelease\Infrastructure\RedisContentReleaseService; use Flowpack\DecoupledContentStore\Utility\GeneratorUtility; use Neos\ContentRepository\Domain\NodeType\NodeTypeConstraintFactory; @@ -44,6 +45,12 @@ class NodeEnumerator */ protected $concurrentBuildLockService; + /** + * @Flow\Inject + * @var NodeRenderingExtensionManager + */ + protected $nodeRenderingExtensionManager; + /** * @Flow\InjectConfiguration("nodeRendering.nodeTypeWhitelist") * @var string @@ -63,9 +70,10 @@ public function enumerateAndStoreInRedis(?Site $site, ContentReleaseLogger $cont foreach (GeneratorUtility::createArrayBatch($this->enumerateAll($site, $contentReleaseLogger, $newMetadata->getWorkspaceName()), 100) as $enumeration) { $this->concurrentBuildLockService->assertNoOtherContentReleaseWasStarted($releaseIdentifier); // $enumeration is an array of EnumeratedNode, with at most 100 elements in it. - // TODO: EXTENSION POINT HERE, TO ADD ADDITIONAL ENUMERATIONS (.metadata.json f.e.) - // TODO: not yet fully sure how to handle Enumeration + $this->redisEnumerationRepository->addDocumentNodesToEnumeration($releaseIdentifier, ...$enumeration); + + // DEPRECATED: use extensions.documentRenderers.[...].enumeratorClassName instead foreach ($enumeration as $enumeratedNode) { $this->emitNodeEnumerated($enumeratedNode, $releaseIdentifier, $contentReleaseLogger); } @@ -81,7 +89,7 @@ private function enumerateAll(?Site $site, ContentReleaseLogger $contentReleaseL $nodeTypeWhitelist = $this->nodeTypeConstraintFactory->parseFilterString($this->nodeTypeWhitelist); - $queueSite = function (Site $site) use ($combinator, &$documentNodeVariantsToRender, $nodeTypeWhitelist, $contentReleaseLogger, $workspaceName) { + $queueSite = function (Site $site) use ($combinator, $nodeTypeWhitelist, $contentReleaseLogger, $workspaceName) { $contentReleaseLogger->debug('Publishing site', [ 'name' => $site->getName(), 'domain' => $site->getFirstActiveDomain() @@ -97,12 +105,15 @@ private function enumerateAll(?Site $site, ContentReleaseLogger $contentReleaseL $contextPath = $documentNode->getContextPath(); if ($nodeTypeWhitelist->matches(NodeTypeName::fromString($documentNode->getNodeType()->getName()))) { - - $contentReleaseLogger->debug('Registering node for publishing', [ - 'node' => $contextPath - ]); - - yield EnumeratedNode::fromNode($documentNode); + foreach ($this->nodeRenderingExtensionManager->enumerateDocumentNode($documentNode) as $enumeratedNode) { + $contentReleaseLogger->debug('Registering node for publishing', [ + 'node' => $contextPath, + 'renderer' => $enumeratedNode->rendererId, + 'arguments' => $enumeratedNode->getArguments(), + ]); + + yield $enumeratedNode; + } } else { $contentReleaseLogger->debug('Skipping node from publishing, because it did not match the configured nodeTypeWhitelist', [ 'node' => $contextPath, @@ -127,6 +138,8 @@ private function enumerateAll(?Site $site, ContentReleaseLogger $contentReleaseL * * This signal can be used to add additional EnumeratedNode entries (e.g. with added arguments for pagination or filters) based on the given node. * + * DEPRECATED: use extensions.documentRenderers.[...].enumeratorClassName instead + * * @param EnumeratedNode $enumeratedNode * @param ContentReleaseIdentifier $releaseIdentifier * @param ContentReleaseLogger $contentReleaseLogger diff --git a/Classes/NodeRendering/Extensibility/DocumentEnumeratorInterface.php b/Classes/NodeRendering/Extensibility/DocumentEnumeratorInterface.php new file mode 100644 index 0000000..1f7e962 --- /dev/null +++ b/Classes/NodeRendering/Extensibility/DocumentEnumeratorInterface.php @@ -0,0 +1,30 @@ + + */ + public function enumerateDocumentNode(NodeInterface $documentNode): iterable; +} diff --git a/Classes/NodeRendering/Extensibility/DocumentEnumerators/DefaultEnumerator.php b/Classes/NodeRendering/Extensibility/DocumentEnumerators/DefaultEnumerator.php new file mode 100644 index 0000000..e680036 --- /dev/null +++ b/Classes/NodeRendering/Extensibility/DocumentEnumerators/DefaultEnumerator.php @@ -0,0 +1,21 @@ +limit = $options['limit'] ?? throw new \InvalidArgumentException('Missing limit option'); + } + public function enumerateDocumentNode(NodeInterface $documentNode): iterable + { + if ($this->i++ >= $this->limit) { + return []; + } + + return [ + EnumeratedNode::fromNode($documentNode), + ]; + } +} diff --git a/Classes/NodeRendering/Extensibility/DocumentRendererInterface.php b/Classes/NodeRendering/Extensibility/DocumentRendererInterface.php new file mode 100644 index 0000000..0df081a --- /dev/null +++ b/Classes/NodeRendering/Extensibility/DocumentRendererInterface.php @@ -0,0 +1,25 @@ +redisContentCacheReader->tryToExtractRenderingForEnumeratedNodeFromContentCache(DocumentNodeCacheKey::fromEnumeratedNode($enumeratedNode)); + } + + public function renderDocumentNodeVariant(NodeInterface $node, EnumeratedNode $enumeratedNode, ContentReleaseLogger $contentReleaseLogger) + { + return $this->documentRenderer->renderDocumentNodeVariant($node, $enumeratedNode->getArguments(), $contentReleaseLogger); + } +} diff --git a/Classes/NodeRendering/Extensibility/NodeRenderingExtensionManager.php b/Classes/NodeRendering/Extensibility/NodeRenderingExtensionManager.php index 7cff988..cde3ae4 100644 --- a/Classes/NodeRendering/Extensibility/NodeRenderingExtensionManager.php +++ b/Classes/NodeRendering/Extensibility/NodeRenderingExtensionManager.php @@ -5,6 +5,7 @@ use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier; use Flowpack\DecoupledContentStore\Core\Infrastructure\ContentReleaseLogger; +use Flowpack\DecoupledContentStore\NodeEnumeration\Domain\Dto\EnumeratedNode; use Flowpack\DecoupledContentStore\NodeRendering\Dto\RenderedDocumentFromContentCache; use Neos\Flow\Annotations as Flow; use Flowpack\DecoupledContentStore\NodeRendering\Dto\DocumentNodeCacheValues; @@ -17,6 +18,30 @@ class NodeRenderingExtensionManager { + /** + * @Flow\InjectConfiguration("extensions.documentRenderers") + * @var array + */ + protected $configuredDocumentRenderers; + + /** + * cache of the instantiated objects which are configured at {@see configuredDocumentRenderers}.enumeratorClassName + * @var DocumentEnumeratorInterface[] + */ + protected $documentEnumerators; + + /** + * cache of the instantiated objects which are configured at {@see configuredDocumentRenderers}.rendererClassName + * @var DocumentRendererInterface[] + */ + protected $documentRenderers; + + /** + * cache of the instantiated objects which are configured at {@see configuredDocumentRenderers}.contentReleaseWriters + * @var ContentReleaseWriterInterface[][] + */ + protected $contentReleaseWriters; + /** * @Flow\InjectConfiguration("extensions.documentMetadataGenerators") * @var array @@ -30,16 +55,44 @@ class NodeRenderingExtensionManager protected $documentMetadataGenerators; /** - * @Flow\InjectConfiguration("extensions.contentReleaseWriters") - * @var array + * @return iterable */ - protected $configuredContentReleaseWriters; + public function enumerateDocumentNode(NodeInterface $documentNode): iterable + { + if (!isset($this->documentEnumerators)) { + $this->documentEnumerators = self::instantiateExtensions($this->configuredDocumentRenderers, DocumentEnumeratorInterface::class, classNameKey: 'enumeratorClassName', optionsKey: 'enumeratorOptions', preserveKey: true); + } + foreach ($this->documentEnumerators as $rendererId => $documentEnumerator) { + foreach ($documentEnumerator->enumerateDocumentNode($documentNode) as $enumeratedNode) { + assert($enumeratedNode instanceof EnumeratedNode); + // we enforce the renderer ID here for the returned enumerated node + yield $enumeratedNode->withRendererId($rendererId); + } + } + } - /** - * cache of the instanciated objects which are configured at {@see $configuredContentReleaseWriters} - * @var ContentReleaseWriterInterface[] - */ - protected $contentReleaseWriters; + public function tryToExtractRenderingForEnumeratedNodeFromContentCache(EnumeratedNode $enumeratedNode): RenderedDocumentFromContentCache + { + return $this->rendererFor($enumeratedNode) + ->tryToExtractRenderingForEnumeratedNodeFromContentCache($enumeratedNode); + } + + public function renderDocumentNodeVariant(NodeInterface $node, EnumeratedNode $enumeratedNode, ContentReleaseLogger $contentReleaseLogger) + { + return $this->rendererFor($enumeratedNode) + ->renderDocumentNodeVariant($node, $enumeratedNode, $contentReleaseLogger); + } + + protected function rendererFor(EnumeratedNode $enumeratedNode): DocumentRendererInterface + { + if (!isset($this->documentEnumerators)) { + $this->documentRenderers = self::instantiateExtensions($this->configuredDocumentRenderers, DocumentRendererInterface::class, classNameKey: 'rendererClassName', preserveKey: true); + } + if (!array_key_exists($enumeratedNode->rendererId, $this->documentRenderers)) { + throw new \RuntimeException('No renderer found for renderer ID ' . $enumeratedNode->rendererId . ' - should never happen!'); + } + return $this->documentRenderers[$enumeratedNode->rendererId]; + } /** * Execute Document Metadata Generators to modify $cacheValues @@ -68,31 +121,39 @@ public function runDocumentMetadataGenerators(NodeInterface $node, array $argume * @param ContentReleaseIdentifier $contentReleaseIdentifier * @param RenderedDocumentFromContentCache $renderedDocumentFromContentCache */ - public function addRenderedDocumentToContentRelease(ContentReleaseIdentifier $contentReleaseIdentifier, RenderedDocumentFromContentCache $renderedDocumentFromContentCache, ContentReleaseLogger $logger): void + public function addRenderedDocumentToContentRelease(ContentReleaseIdentifier $contentReleaseIdentifier, EnumeratedNode $enumeratedNode, RenderedDocumentFromContentCache $renderedDocumentFromContentCache, ContentReleaseLogger $logger): void { - if (!isset($this->contentReleaseWriters)) { - $this->contentReleaseWriters = self::instantiateExtensions($this->configuredContentReleaseWriters, ContentReleaseWriterInterface::class); + if (!isset($this->contentReleaseWriters[$enumeratedNode->rendererId])) { + $this->contentReleaseWriters[$enumeratedNode->rendererId] = self::instantiateExtensions($this->configuredDocumentRenderers[$enumeratedNode->rendererId]['contentReleaseWriters'], ContentReleaseWriterInterface::class); } - foreach ($this->contentReleaseWriters as $contentReleaseWriter) { + foreach ($this->contentReleaseWriters[$enumeratedNode->rendererId] as $contentReleaseWriter) { assert($contentReleaseWriter instanceof ContentReleaseWriterInterface); $contentReleaseWriter->processRenderedDocument($contentReleaseIdentifier, $renderedDocumentFromContentCache, $logger); } } - private static function instantiateExtensions(array $configuration, string $extensionInterfaceName): array + private static function instantiateExtensions(array $configuration, string $extensionInterfaceName, string $classNameKey = 'className', string|null $optionsKey = null, bool $preserveKey = false): array { $instantiatedExtensions = []; - foreach ($configuration as $extensionConfig) { + foreach ($configuration as $k => $extensionConfig) { if (!is_array($extensionConfig)) { continue; } - $className = $extensionConfig['className']; - $instance = new $className(); + $className = $extensionConfig[$classNameKey]; + if ($optionsKey !== null) { + $instance = new $className($extensionConfig[$optionsKey] ?? []); + } else { + $instance = new $className(); + } if (!($instance instanceof $extensionInterfaceName)) { throw new \RuntimeException('Extension ' . get_class($instance) . ' does not implement ' . $extensionInterfaceName); } - $instantiatedExtensions[] = $instance; + if ($preserveKey) { + $instantiatedExtensions[$k] = $instance; + } else { + $instantiatedExtensions[] = $instance; + } } return $instantiatedExtensions; } diff --git a/Classes/NodeRendering/NodeRenderOrchestrator.php b/Classes/NodeRendering/NodeRenderOrchestrator.php index d5e3f10..c4b2b75 100644 --- a/Classes/NodeRendering/NodeRenderOrchestrator.php +++ b/Classes/NodeRendering/NodeRenderOrchestrator.php @@ -6,10 +6,8 @@ use Flowpack\DecoupledContentStore\Core\ConcurrentBuildLockService; use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\RedisInstanceIdentifier; -use Flowpack\DecoupledContentStore\NodeRendering\Dto\DocumentNodeCacheKey; use Flowpack\DecoupledContentStore\NodeRendering\Dto\RenderingStatistics; use Flowpack\DecoupledContentStore\NodeRendering\Extensibility\NodeRenderingExtensionManager; -use Flowpack\DecoupledContentStore\NodeRendering\Infrastructure\RedisContentCacheReader; use Flowpack\DecoupledContentStore\NodeRendering\Infrastructure\RedisRenderingErrorManager; use Flowpack\DecoupledContentStore\NodeRendering\Infrastructure\RedisRenderingStatisticsStore; use Flowpack\DecoupledContentStore\NodeRendering\ProcessEvents\ExitEvent; @@ -64,12 +62,6 @@ class NodeRenderOrchestrator */ protected $redisRenderingQueue; - /** - * @Flow\Inject - * @var RedisContentCacheReader - */ - protected $redisContentCacheReader; - /** * @Flow\Inject * @var RedisRenderingErrorManager @@ -156,7 +148,7 @@ public function renderContentRelease(ContentReleaseIdentifier $contentReleaseIde foreach ($currentEnumeration as $enumeratedNode) { assert($enumeratedNode instanceof EnumeratedNode); - $renderedDocumentFromContentCache = $this->redisContentCacheReader->tryToExtractRenderingForEnumeratedNodeFromContentCache(DocumentNodeCacheKey::fromEnumeratedNode($enumeratedNode)); + $renderedDocumentFromContentCache = $this->nodeRenderingExtensionManager->tryToExtractRenderingForEnumeratedNodeFromContentCache($enumeratedNode); if ($renderedDocumentFromContentCache->isComplete()) { $contentReleaseLogger->debug( 'Node fully rendered, adding to content release', @@ -164,7 +156,7 @@ public function renderContentRelease(ContentReleaseIdentifier $contentReleaseIde ); // NOTE: Eventually consistent (TODO describe) // If wanted more fully consistent, move to bottom.... - $this->nodeRenderingExtensionManager->addRenderedDocumentToContentRelease($contentReleaseIdentifier, $renderedDocumentFromContentCache, $contentReleaseLogger); + $this->nodeRenderingExtensionManager->addRenderedDocumentToContentRelease($contentReleaseIdentifier, $enumeratedNode, $renderedDocumentFromContentCache, $contentReleaseLogger); } else { $contentReleaseLogger->debug( 'Scheduling rendering for Node, as it was not found or its content is incomplete: ' diff --git a/Classes/NodeRendering/NodeRenderer.php b/Classes/NodeRendering/NodeRenderer.php index dd786c2..6436e96 100644 --- a/Classes/NodeRendering/NodeRenderer.php +++ b/Classes/NodeRendering/NodeRenderer.php @@ -6,6 +6,7 @@ use Flowpack\DecoupledContentStore\ContentReleaseManager; use Flowpack\DecoupledContentStore\Core\ConcurrentBuildLockService; +use Flowpack\DecoupledContentStore\NodeRendering\Extensibility\NodeRenderingExtensionManager; use Flowpack\DecoupledContentStore\NodeRendering\ProcessEvents\DocumentRenderedEvent; use Flowpack\DecoupledContentStore\NodeRendering\ProcessEvents\ExitEvent; use Flowpack\DecoupledContentStore\NodeRendering\ProcessEvents\QueueEmptyEvent; @@ -51,13 +52,6 @@ */ class NodeRenderer { - - /** - * @Flow\Inject - * @var DocumentRenderer - */ - protected $documentRenderer; - /** * @Flow\Inject * @var RedisRenderingQueue @@ -107,6 +101,11 @@ class NodeRenderer */ protected $concurrentBuildLockService; + /** + * @Flow\Inject + * @var NodeRenderingExtensionManager + */ + protected $nodeRenderingExtensionManager; public function render(ContentReleaseIdentifier $contentReleaseIdentifier, ContentReleaseLogger $contentReleaseLogger, RendererIdentifier $rendererIdentifier) { @@ -199,7 +198,7 @@ protected function renderDocumentNodeVariant(EnumeratedNode $enumeratedNode, Con 'arguments' => $enumeratedNode->getArguments() ]); - $this->documentRenderer->renderDocumentNodeVariant($node, $enumeratedNode->getArguments(), $contentReleaseLogger); + $this->nodeRenderingExtensionManager->renderDocumentNodeVariant($node, $enumeratedNode, $contentReleaseLogger); } // NOTE: we do not abort rendering directly, when we encounter any error, but we try to render // all pages in the full iteration (and then, if errors exist, we stop). diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml index e4e4907..b364acc 100644 --- a/Configuration/Settings.yaml +++ b/Configuration/Settings.yaml @@ -47,6 +47,28 @@ Flowpack: extensions: + + # Extend node rendering by deciding HOW the rendering should happen. This is done during the Enumeration phase + # at the beginning: for each document node (=which matches nodeTypeWhitelist), all documentNodeEnumerators + # are called, and they can decide which DocumentNodeRenderer should be used by creating appropriate + # instances of EnumeratedNode and yielding them. + documentRenderers: + htmlViaFusion: + enumeratorClassName: Flowpack\DecoupledContentStore\NodeRendering\Extensibility\DocumentEnumerators\DefaultEnumerator + #enumeratorClassName: Flowpack\DecoupledContentStore\NodeRendering\Extensibility\DocumentEnumerators\LimitEnumerator + #enumeratorOptions: + # limit: 20 + rendererClassName: Flowpack\DecoupledContentStore\NodeRendering\Extensibility\DocumentRenderers\FusionHtmlRenderer + + # Register additional content release writers, being called for every finished node which should be added + # to the content release. + # (must implement ContentReleaseWriterInterface) + contentReleaseWriters: + gzipCompressed: + className: Flowpack\DecoupledContentStore\NodeRendering\Extensibility\ContentReleaseWriters\GzipWriter + legacy: + className: Flowpack\DecoupledContentStore\NodeRendering\Extensibility\ContentReleaseWriters\LegacyWriter + # Extend Node Rendering by generating additional data for each rendering, interleaved with the content cache. # # (must implement DocumentMetadataGeneratorInterface) @@ -54,15 +76,6 @@ Flowpack: # className: Fully\Qualified\Classname documentMetadataGenerators: [] - # Register additional content release writers, being called for every finished node which should be added - # to the content release. - # (must implement ContentReleaseWriterInterface) - contentReleaseWriters: - gzipCompressed: - className: Flowpack\DecoupledContentStore\NodeRendering\Extensibility\ContentReleaseWriters\GzipWriter - legacy: - className: Flowpack\DecoupledContentStore\NodeRendering\Extensibility\ContentReleaseWriters\LegacyWriter - redisKeyPostfixesForEachRelease: # each content release starts with a fixed prefix "contentRelease:[releaseId]:", and afterwards # follows a part which needs to be registered here. This is needed for synchronization between different From ee9eff963cc0eecb056918a2d367432f86576a24 Mon Sep 17 00:00:00 2001 From: Sebastian Kurfuerst Date: Thu, 17 Jul 2025 08:42:33 +0200 Subject: [PATCH 02/10] REFACTOR: Extract URI Generation for node into separate service ... because this is useful in different rendering scenarios --- .../NodeRendering/NodeRenderingUriService.php | 158 +++++++++++++++++ .../NodeRendering/Render/DocumentRenderer.php | 159 +----------------- 2 files changed, 167 insertions(+), 150 deletions(-) create mode 100644 Classes/NodeRendering/NodeRenderingUriService.php diff --git a/Classes/NodeRendering/NodeRenderingUriService.php b/Classes/NodeRendering/NodeRenderingUriService.php new file mode 100644 index 0000000..40563d7 --- /dev/null +++ b/Classes/NodeRendering/NodeRenderingUriService.php @@ -0,0 +1,158 @@ +getContext()->getCurrentSite(); + if (!$currentSite->hasActiveDomains()) { + throw new Exception(sprintf("Site %s has no active domain", $currentSite->getNodeName()), 1666684522); + } + $primaryDomain = $currentSite->getPrimaryDomain(); + if ((string)$primaryDomain->getScheme() === '') { + throw new Exception(sprintf("Domain %s for site %s has no scheme defined", $primaryDomain->getHostname(), $currentSite->getNodeName()), 1666684523); + } + + // HINT: We cannot use a static URL here, but instead need to use an URL of the current site. + // This is changed from the the old behavior, where we have changed the LinkingService in LinkingServiceAspect, + // to properly generate the domain part of the routes - and this relies on the proper ControllerContext URI path. + $baseControllerContext = $this->buildControllerContextAndSetBaseUri($primaryDomain->__toString(), $node, $arguments); + $format = $arguments['@format'] ?? 'html'; + $uri = $this->linkingService->createNodeUri($baseControllerContext, $node, null, $format, true, $arguments, '', false, [], false); + return self::removeQueryPartFromUri($uri); + } + + /** + * @param string $uri + * @param NodeInterface $node + * @param array $arguments + * @return ControllerContext + */ + public function buildControllerContextAndSetBaseUri(string $uri, NodeInterface $node, array $arguments = []) + { + $request = $this->buildFakeRequest($uri, $node); + if (isset($arguments['@format'])) { + $request->setFormat($arguments['@format']); + } + + // NASTY SIDE-EFFECT: we not only build the controller context, but we also need to inject the "current" base URL to BaseUriProvider, + // as this is now (Flow 6.x) used in the UriBuilder to determine the domain. + $baseUri = rtrim(RequestInformationHelper::generateBaseUri($request->getHttpRequest())->__toString(), '/'); + ObjectAccess::setProperty($this->baseUriProvider, 'configuredBaseUri', $baseUri, true); + + ObjectAccess::setProperty($this->securityContext, 'initialized', true, true); + $this->securityContext->setRequest($request); + $uriBuilder = $this->uriBuilderForRequest($request); + + return new ControllerContext( + $request, + new ActionResponse(), + new Arguments([]), + $uriBuilder + ); + } + + /** + * @param string $uri + * @param NodeInterface $node + * @return ActionRequest + */ + protected function buildFakeRequest($uri, NodeInterface $node): ActionRequest + { + $_SERVER['FLOW_REWRITEURLS'] = '1'; + + $httpRequest = new ServerRequest('GET', $uri); + $routingParameters = RouteParameters::createEmpty()->withParameter('requestUriHost', $httpRequest->getUri()->getHost()); + $httpRequest = $httpRequest->withAttribute(ServerRequestAttributes::ROUTING_PARAMETERS, $routingParameters); + + $request = ActionRequest::fromHttpRequest($httpRequest); + $request->setControllerObjectName('Neos\Neos\Controller\Frontend\NodeController'); + $request->setControllerActionName('show'); + $request->setFormat('html'); + $request->setArgument('node', $node->getContextPath()); + + return $request; + } + + + /** + * @param ActionRequest $request + * @return UriBuilder + * @throws \Neos\Utility\Exception\PropertyNotAccessibleException + */ + protected function uriBuilderForRequest(ActionRequest $request): UriBuilder + { + $uriBuilder = new UriBuilder(); + $uriBuilder->setRequest($request); + + $routesConfiguration = $this->configurationManager->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_ROUTES); + $router = ObjectAccess::getProperty($uriBuilder, 'router', true); + + $router->setRoutesConfiguration($routesConfiguration); + + return $uriBuilder; + } + + + /** + * @param string $uri + * @return string + */ + protected static function removeQueryPartFromUri($uri) + { + $uriData = explode('?', $uri); + + return $uriData[0]; + } +} diff --git a/Classes/NodeRendering/Render/DocumentRenderer.php b/Classes/NodeRendering/Render/DocumentRenderer.php index bf9e976..3a1513c 100644 --- a/Classes/NodeRendering/Render/DocumentRenderer.php +++ b/Classes/NodeRendering/Render/DocumentRenderer.php @@ -5,6 +5,7 @@ use Flowpack\DecoupledContentStore\Aspects\CacheUrlMappingAspect; use Flowpack\DecoupledContentStore\Exception; use Flowpack\DecoupledContentStore\Core\Infrastructure\ContentReleaseLogger; +use Flowpack\DecoupledContentStore\NodeRendering\NodeRenderingUriService; use Flowpack\DecoupledContentStore\Transfer\Resource\Target\MultisiteFileSystemSymlinkTarget; use GuzzleHttp\Psr7\ServerRequest; use Neos\Flow\Annotations as Flow; @@ -38,6 +39,12 @@ class DocumentRenderer */ protected $useRelativeResourceUris; + /** + * @Flow\Inject + * @var NodeRenderingUriService + */ + protected $nodeRenderingUriService; + /** * Add HTTP message if rendering full content * @@ -52,51 +59,12 @@ class DocumentRenderer */ protected $fusionView; - /** - * @Flow\Inject - * @var SecurityContext - */ - protected $securityContext; - - /** - * @Flow\Inject - * @var ConfigurationManager - */ - protected $configurationManager; - /** * @Flow\Inject(lazy=false) * @var ResourceManager */ protected $resourceManager; - /** - * @Flow\Inject - * @var \Neos\Neos\Service\LinkingService - */ - protected $linkingService; - - /** - * NOTE: we need to use EAGER injection here, because in buildControllerContextAndSetBaseUri(), we - * directly set a property of the BaseUriProvider using ObjectAccess::setProperty() with forceDirectAccess. - * - * @Flow\Inject(lazy=false) - * @var BaseUriProvider - */ - protected $baseUriProvider; - - /** - * @var \Neos\Flow\Mvc\Routing\Router - * @Flow\Inject - */ - protected $router; - - /** - * @Flow\Inject - * @var \Neos\Flow\Property\PropertyMapper - */ - protected $propertyMapper; - /** * @Flow\Inject * @var CacheUrlMappingAspect @@ -119,7 +87,7 @@ class DocumentRenderer public function renderDocumentNodeVariant(NodeInterface $node, array $arguments, ContentReleaseLogger $contentReleaseLogger): string { $this->cacheUrlMappingAspect->beforeDocumentRendering($contentReleaseLogger); - $nodeUri = $this->buildNodeUri($node, $arguments); + $nodeUri = $this->nodeRenderingUriService->buildNodeUri($node, $arguments); try { $arguments['node'] = $node->getContextPath(); @@ -131,75 +99,6 @@ public function renderDocumentNodeVariant(NodeInterface $node, array $arguments, } } - /** - * @param string $uri - * @param NodeInterface $node - * @param array $arguments - * @return ControllerContext - */ - protected function buildControllerContextAndSetBaseUri(string $uri, NodeInterface $node, array $arguments = []) - { - $request = $this->getRequest($uri, $node); - if (isset($arguments['@format'])) { - $request->setFormat($arguments['@format']); - } - - // NASTY SIDE-EFFECT: we not only build the controller context, but we also need to inject the "current" base URL to BaseUriProvider, - // as this is now (Flow 6.x) used in the UriBuilder to determine the domain. - $baseUri = rtrim(RequestInformationHelper::generateBaseUri($request->getHttpRequest()), '/'); - ObjectAccess::setProperty($this->baseUriProvider, 'configuredBaseUri', $baseUri, true); - - ObjectAccess::setProperty($this->securityContext, 'initialized', true, true); - $this->securityContext->setRequest($request); - $uriBuilder = $this->getUriBuilder($request); - - return new ControllerContext( - $request, - new ActionResponse(), - new Arguments([]), - $uriBuilder - ); - } - - /** - * @param ActionRequest $request - * @return UriBuilder - * @throws \Neos\Utility\Exception\PropertyNotAccessibleException - */ - protected function getUriBuilder($request) - { - $uriBuilder = new UriBuilder(); - $uriBuilder->setRequest($request); - - $routesConfiguration = $this->configurationManager->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_ROUTES); - $router = ObjectAccess::getProperty($uriBuilder, 'router', true); - - $router->setRoutesConfiguration($routesConfiguration); - - return $uriBuilder; - } - - /** - * @param string $uri - * @param NodeInterface $node - * @return ActionRequest - */ - protected function getRequest($uri, NodeInterface $node) - { - $_SERVER['FLOW_REWRITEURLS'] = '1'; - - $httpRequest = new ServerRequest('GET', $uri); - $routingParameters = RouteParameters::createEmpty()->withParameter('requestUriHost', $httpRequest->getUri()->getHost()); - $httpRequest = $httpRequest->withAttribute(ServerRequestAttributes::ROUTING_PARAMETERS, $routingParameters); - - $request = ActionRequest::fromHttpRequest($httpRequest); - $request->setControllerObjectName('Neos\Neos\Controller\Frontend\NodeController'); - $request->setControllerActionName('show'); - $request->setFormat('html'); - $request->setArgument('node', $node->getContextPath()); - - return $request; - } /** * Render the view of a document node @@ -231,7 +130,7 @@ protected function renderDocumentView(NodeInterface $node, $uri, array $requestA $contentReleaseLogger->info('Rendering document for URI ' . $uri, ['baseUri' => $baseUri]); - $controllerContext = $this->buildControllerContextAndSetBaseUri($uri, $node, $requestArguments); + $controllerContext = $this->nodeRenderingUriService->buildControllerContextAndSetBaseUri($uri, $node, $requestArguments); /** @var ActionRequest $request */ $request = $controllerContext->getRequest(); $request->setArguments($requestArguments); @@ -282,34 +181,6 @@ private static function wrapInHttpMessage(string $output, ActionResponse $respon return "HTTP/1.1" . (empty($headerLines) ? "\r\n" : implode("\r\n", $headerLines)) . "\r\n" . $output; } - - /** - * @param NodeInterface $node - * @param array $arguments - * @return string The resolved URI for the given node - * @throws \Exception - */ - protected function buildNodeUri(NodeInterface $node, array $arguments) - { - /** @var Site $currentSite */ - $currentSite = $node->getContext()->getCurrentSite(); - if (!$currentSite->hasActiveDomains()) { - throw new Exception(sprintf("Site %s has no active domain", $currentSite->getNodeName()), 1666684522); - } - $primaryDomain = $currentSite->getPrimaryDomain(); - if ((string)$primaryDomain->getScheme() === '') { - throw new Exception(sprintf("Domain %s for site %s has no scheme defined", $primaryDomain->getHostname(), $currentSite->getNodeName()), 1666684523); - } - - // HINT: We cannot use a static URL here, but instead need to use an URL of the current site. - // This is changed from the the old behavior, where we have changed the LinkingService in LinkingServiceAspect, - // to properly generate the domain part of the routes - and this relies on the proper ControllerContext URI path. - $baseControllerContext = $this->buildControllerContextAndSetBaseUri($primaryDomain->__toString(), $node, $arguments); - $format = $arguments['@format'] ?? 'html'; - $uri = $this->linkingService->createNodeUri($baseControllerContext, $node, null, $format, true, $arguments, '', false, [], false); - return $this->removeQueryPartFromUri($uri); - } - /** * @return bool */ @@ -318,17 +189,6 @@ public function isRendering() return $this->isRendering; } - /** - * @param string $uri - * @return string - */ - protected function removeQueryPartFromUri($uri) - { - $uriData = explode('?', $uri); - - return $uriData[0]; - } - public function disableCache() { $this->fusionView->disableCache(); @@ -338,5 +198,4 @@ public function enableCache() { $this->fusionView->enableCache(); } - } From 9e6f64467c10780b35a14e66f33c8bf2fc130ed3 Mon Sep 17 00:00:00 2001 From: Sebastian Kurfuerst Date: Thu, 17 Jul 2025 15:16:04 +0200 Subject: [PATCH 03/10] TASK: update README with upgrade instructions for v2 --- README.md | 72 ++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 58 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 7b72226..4a4be2d 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,55 @@ way, based on the learnings of the first iteration. Especially the robustness ha NOTE: There still exists the pull request https://github.com/Flowpack/Flowpack.DecoupledContentStore/pull/42 containing various improvements, which also should go to 2.x / main branch. +### Updating from 1.x to 2.x (not yet released) + +You need to adjust the following things when updating from DecoupledContentStore 1.x to 2.x: + +NEW FEATURES / IMPROVEMENTS: + + + +UPDATING: + +**Settings.yaml - contentReleaseWriters must be configured differently. + +OLD: +```yaml + +Flowpack: + DecoupledContentStore: + extensions: + contentReleaseWriters: + gzipCompressed: + className: Flowpack\DecoupledContentStore\NodeRendering\Extensibility\ContentReleaseWriters\GzipWriter + legacy: + className: Flowpack\DecoupledContentStore\NodeRendering\Extensibility\ContentReleaseWriters\LegacyWriter + +``` + +NEW: + +```yaml +Flowpack: + DecoupledContentStore: + extensions: + # Decide how node rendering should happen. + documentRenderers: + htmlViaFusion: + # ... the other config matches the old behavior ... + + # Register additional content release writers, being called for every finished node which should be added + # to the content release. + # (must implement ContentReleaseWriterInterface) + contentReleaseWriters: + gzipCompressed: + className: Flowpack\DecoupledContentStore\NodeRendering\Extensibility\ContentReleaseWriters\GzipWriter + legacy: + className: Flowpack\DecoupledContentStore\NodeRendering\Extensibility\ContentReleaseWriters\LegacyWriter + + +``` + ## What does it do? The Content Store package publishes content from Neos to a Redis database as @@ -300,6 +349,15 @@ eventually executed, because that's prunner's job. Crafting a custom `pipelines.yml` is the main extension point for doing additional work (f.e. additional enumeration or rendering). +### Custom Rendering + +(NEW with v2) + +DecoupledContentStore v1 was specifically tied to Fusion as rendering engine and the Neos Content cache. +This has changed in V2, where **different renderings** of a given document can be instantiated. + +(TODO EXPLAIN IN DETAIL) + ### Custom Document Metadata, integrated with the Content Cache Sometimes, you need to build additional data structures for every individual document. Ideally, you'll want this @@ -528,20 +586,6 @@ error. Then, you can inspect the testing database and manually reproduce the bug Additionally, `-vvv` is a helpful CLI flag (extra-verbose) - this displays the full exception stack trace in case of errors. -## TODO - -- clean up of old content releases - - in Content Store / Redis -- generate the old content format -- (SK) error handling tests -- force-switch possibility -- (AM) UI -- check for TODOs :) - -## Missing Features from old - -data-url-next-page (or so) not supported - ## License GPL v3 From ceac054975ab3e1f571bbaabb655e6a95bb5d1cc Mon Sep 17 00:00:00 2001 From: Sebastian Kurfuerst Date: Thu, 17 Jul 2025 15:16:44 +0200 Subject: [PATCH 04/10] FEATURE: Support options for ContentReleaseWriters --- .../Extensibility/NodeRenderingExtensionManager.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Classes/NodeRendering/Extensibility/NodeRenderingExtensionManager.php b/Classes/NodeRendering/Extensibility/NodeRenderingExtensionManager.php index cde3ae4..98710cf 100644 --- a/Classes/NodeRendering/Extensibility/NodeRenderingExtensionManager.php +++ b/Classes/NodeRendering/Extensibility/NodeRenderingExtensionManager.php @@ -124,7 +124,7 @@ public function runDocumentMetadataGenerators(NodeInterface $node, array $argume public function addRenderedDocumentToContentRelease(ContentReleaseIdentifier $contentReleaseIdentifier, EnumeratedNode $enumeratedNode, RenderedDocumentFromContentCache $renderedDocumentFromContentCache, ContentReleaseLogger $logger): void { if (!isset($this->contentReleaseWriters[$enumeratedNode->rendererId])) { - $this->contentReleaseWriters[$enumeratedNode->rendererId] = self::instantiateExtensions($this->configuredDocumentRenderers[$enumeratedNode->rendererId]['contentReleaseWriters'], ContentReleaseWriterInterface::class); + $this->contentReleaseWriters[$enumeratedNode->rendererId] = self::instantiateExtensions($this->configuredDocumentRenderers[$enumeratedNode->rendererId]['contentReleaseWriters'], ContentReleaseWriterInterface::class, optionsKey: 'options'); } foreach ($this->contentReleaseWriters[$enumeratedNode->rendererId] as $contentReleaseWriter) { assert($contentReleaseWriter instanceof ContentReleaseWriterInterface); From 65ef99c090f8f6fd7ab452ef6d7db52d78f39589 Mon Sep 17 00:00:00 2001 From: Sebastian Kurfuerst Date: Thu, 17 Jul 2025 15:17:00 +0200 Subject: [PATCH 05/10] TASK: minor type fixes for DocumentRenderers --- .../NodeRendering/Extensibility/DocumentRendererInterface.php | 2 +- .../Extensibility/DocumentRenderers/FusionHtmlRenderer.php | 4 ++-- .../Extensibility/NodeRenderingExtensionManager.php | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Classes/NodeRendering/Extensibility/DocumentRendererInterface.php b/Classes/NodeRendering/Extensibility/DocumentRendererInterface.php index 0df081a..c283b8e 100644 --- a/Classes/NodeRendering/Extensibility/DocumentRendererInterface.php +++ b/Classes/NodeRendering/Extensibility/DocumentRendererInterface.php @@ -21,5 +21,5 @@ interface DocumentRendererInterface { public function tryToExtractRenderingForEnumeratedNodeFromContentCache(EnumeratedNode $enumeratedNode): RenderedDocumentFromContentCache; - public function renderDocumentNodeVariant(NodeInterface $node, EnumeratedNode $enumeratedNode, ContentReleaseLogger $contentReleaseLogger); + public function renderDocumentNodeVariant(NodeInterface $node, EnumeratedNode $enumeratedNode, ContentReleaseLogger $contentReleaseLogger): void; } diff --git a/Classes/NodeRendering/Extensibility/DocumentRenderers/FusionHtmlRenderer.php b/Classes/NodeRendering/Extensibility/DocumentRenderers/FusionHtmlRenderer.php index 1d2a362..da499bf 100644 --- a/Classes/NodeRendering/Extensibility/DocumentRenderers/FusionHtmlRenderer.php +++ b/Classes/NodeRendering/Extensibility/DocumentRenderers/FusionHtmlRenderer.php @@ -33,8 +33,8 @@ public function tryToExtractRenderingForEnumeratedNodeFromContentCache(Enumerate return $this->redisContentCacheReader->tryToExtractRenderingForEnumeratedNodeFromContentCache(DocumentNodeCacheKey::fromEnumeratedNode($enumeratedNode)); } - public function renderDocumentNodeVariant(NodeInterface $node, EnumeratedNode $enumeratedNode, ContentReleaseLogger $contentReleaseLogger) + public function renderDocumentNodeVariant(NodeInterface $node, EnumeratedNode $enumeratedNode, ContentReleaseLogger $contentReleaseLogger): void { - return $this->documentRenderer->renderDocumentNodeVariant($node, $enumeratedNode->getArguments(), $contentReleaseLogger); + $this->documentRenderer->renderDocumentNodeVariant($node, $enumeratedNode->getArguments(), $contentReleaseLogger); } } diff --git a/Classes/NodeRendering/Extensibility/NodeRenderingExtensionManager.php b/Classes/NodeRendering/Extensibility/NodeRenderingExtensionManager.php index 98710cf..8d3a3b3 100644 --- a/Classes/NodeRendering/Extensibility/NodeRenderingExtensionManager.php +++ b/Classes/NodeRendering/Extensibility/NodeRenderingExtensionManager.php @@ -77,9 +77,9 @@ public function tryToExtractRenderingForEnumeratedNodeFromContentCache(Enumerate ->tryToExtractRenderingForEnumeratedNodeFromContentCache($enumeratedNode); } - public function renderDocumentNodeVariant(NodeInterface $node, EnumeratedNode $enumeratedNode, ContentReleaseLogger $contentReleaseLogger) + public function renderDocumentNodeVariant(NodeInterface $node, EnumeratedNode $enumeratedNode, ContentReleaseLogger $contentReleaseLogger): void { - return $this->rendererFor($enumeratedNode) + $this->rendererFor($enumeratedNode) ->renderDocumentNodeVariant($node, $enumeratedNode, $contentReleaseLogger); } From e5259685d7c27131e4737c41d201f9cd4a531a89 Mon Sep 17 00:00:00 2001 From: Sebastian Kurfuerst Date: Wed, 6 Aug 2025 10:54:42 +0200 Subject: [PATCH 06/10] TASK: improve LimitEnumerator for debugging --- .../DocumentEnumerators/LimitEnumerator.php | 20 +++++++++++++++---- Configuration/Settings.yaml | 1 + 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Classes/NodeRendering/Extensibility/DocumentEnumerators/LimitEnumerator.php b/Classes/NodeRendering/Extensibility/DocumentEnumerators/LimitEnumerator.php index 6a10696..6bae087 100644 --- a/Classes/NodeRendering/Extensibility/DocumentEnumerators/LimitEnumerator.php +++ b/Classes/NodeRendering/Extensibility/DocumentEnumerators/LimitEnumerator.php @@ -13,17 +13,29 @@ class LimitEnumerator implements DocumentEnumeratorInterface { protected int $i = 0; - private int $limit; + private ?int $limit = null; + private ?string $uriPathSegmentFilter = null; public function __construct( array $options = [] ) { - $this->limit = $options['limit'] ?? throw new \InvalidArgumentException('Missing limit option'); + $this->limit = $options['limit'] ?? null; + $this->uriPathSegmentFilter = $options['uriPathSegmentFilter'] ?? null; } public function enumerateDocumentNode(NodeInterface $documentNode): iterable { - if ($this->i++ >= $this->limit) { - return []; + if ($this->uriPathSegmentFilter !== null) { + if (str_contains($documentNode->getProperty('uriPathSegment'), $this->uriPathSegmentFilter) === false) { + return []; + } + } + + // NOTE: Limiting must come LAST, after all other constraints have been evaluated (because we want + // only filtered nodes from the other conditions to be counted against the limit) + if ($this->limit !== null) { + if ($this->i++ >= $this->limit) { + return []; + } } return [ diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml index b364acc..d5a34b1 100644 --- a/Configuration/Settings.yaml +++ b/Configuration/Settings.yaml @@ -57,6 +57,7 @@ Flowpack: enumeratorClassName: Flowpack\DecoupledContentStore\NodeRendering\Extensibility\DocumentEnumerators\DefaultEnumerator #enumeratorClassName: Flowpack\DecoupledContentStore\NodeRendering\Extensibility\DocumentEnumerators\LimitEnumerator #enumeratorOptions: + # uriPathSegmentFilter: vanucci-collection # limit: 20 rendererClassName: Flowpack\DecoupledContentStore\NodeRendering\Extensibility\DocumentRenderers\FusionHtmlRenderer From ef8706e188eced8681952c85a8c133b7e57e469b Mon Sep 17 00:00:00 2001 From: Andreas Sacher Date: Wed, 16 Jul 2025 12:17:17 +0200 Subject: [PATCH 07/10] FEATURE: Add statistics event logging to ContentReleaseLogger --- Classes/BackendUi/BackendUiDataService.php | 4 +-- .../ConsoleStatisticsEventOutput.php | 33 +++++++++++++++++++ .../Infrastructure/ContentReleaseLogger.php | 23 +++++++++---- .../RedisStatisticsEventOutput.php | 27 +++++++++++++++ .../StatisticsEventOutputInterface.php | 11 +++++++ ... => RedisRenderingTimeStatisticsStore.php} | 2 +- .../NodeRendering/NodeRenderOrchestrator.php | 4 +-- Configuration/Settings.yaml | 5 +++ 8 files changed, 98 insertions(+), 11 deletions(-) create mode 100644 Classes/Core/Infrastructure/ConsoleStatisticsEventOutput.php create mode 100644 Classes/Core/Infrastructure/RedisStatisticsEventOutput.php create mode 100644 Classes/Core/Infrastructure/StatisticsEventOutputInterface.php rename Classes/NodeRendering/Infrastructure/{RedisRenderingStatisticsStore.php => RedisRenderingTimeStatisticsStore.php} (99%) diff --git a/Classes/BackendUi/BackendUiDataService.php b/Classes/BackendUi/BackendUiDataService.php index ebea4dd..898a532 100644 --- a/Classes/BackendUi/BackendUiDataService.php +++ b/Classes/BackendUi/BackendUiDataService.php @@ -13,7 +13,7 @@ use Flowpack\DecoupledContentStore\NodeEnumeration\Domain\Repository\RedisEnumerationRepository; use Flowpack\DecoupledContentStore\NodeRendering\Dto\RenderingStatistics; use Flowpack\DecoupledContentStore\NodeRendering\Infrastructure\RedisRenderingErrorManager; -use Flowpack\DecoupledContentStore\NodeRendering\Infrastructure\RedisRenderingStatisticsStore; +use Flowpack\DecoupledContentStore\NodeRendering\Infrastructure\RedisRenderingTimeStatisticsStore; use Flowpack\DecoupledContentStore\PrepareContentRelease\Infrastructure\RedisContentReleaseService; use Flowpack\DecoupledContentStore\ReleaseSwitch\Infrastructure\RedisReleaseSwitchService; use Neos\Flow\Annotations as Flow; @@ -44,7 +44,7 @@ class BackendUiDataService /** * @Flow\Inject - * @var RedisRenderingStatisticsStore + * @var RedisRenderingTimeStatisticsStore */ protected $redisRenderingStatisticsStore; diff --git a/Classes/Core/Infrastructure/ConsoleStatisticsEventOutput.php b/Classes/Core/Infrastructure/ConsoleStatisticsEventOutput.php new file mode 100644 index 0000000..670332c --- /dev/null +++ b/Classes/Core/Infrastructure/ConsoleStatisticsEventOutput.php @@ -0,0 +1,33 @@ +output = $output; + } + + public static function fromConsoleOutput(ConsoleOutput $output): self + { + return static::fromSymfonyOutput($output->getOutput()); + } + + public static function fromSymfonyOutput(OutputInterface $output): self + { + return new static($output); + } + + public function writeEvent(ContentReleaseIdentifier $contentReleaseIdentifier, string $prefix, string $event, array $additionalPayload): void + { + $this->output->writeln($prefix . 'STATISTICS EVENT ' . $event . ($additionalPayload ? ' ' . json_encode($additionalPayload) : '')); + } +} diff --git a/Classes/Core/Infrastructure/ContentReleaseLogger.php b/Classes/Core/Infrastructure/ContentReleaseLogger.php index 5d75c7f..c008d4c 100644 --- a/Classes/Core/Infrastructure/ContentReleaseLogger.php +++ b/Classes/Core/Infrastructure/ContentReleaseLogger.php @@ -14,6 +14,11 @@ class ContentReleaseLogger */ protected $output; + /** + * @var StatisticsEventOutputInterface + */ + protected $statisticsEventOutput; + /** * @var ContentReleaseIdentifier */ @@ -24,10 +29,11 @@ class ContentReleaseLogger */ protected $logPrefix = ''; - protected function __construct(OutputInterface $output, ContentReleaseIdentifier $contentReleaseIdentifier, ?RendererIdentifier $rendererIdentifier) + protected function __construct(OutputInterface $output, ContentReleaseIdentifier $contentReleaseIdentifier, StatisticsEventOutputInterface $statisticsEventOutput, ?RendererIdentifier $rendererIdentifier) { $this->output = $output; $this->contentReleaseIdentifier = $contentReleaseIdentifier; + $this->statisticsEventOutput = $statisticsEventOutput; $this->rendererIdentifier = $rendererIdentifier; $this->logPrefix = ''; @@ -37,14 +43,14 @@ protected function __construct(OutputInterface $output, ContentReleaseIdentifier } - public static function fromConsoleOutput(ConsoleOutput $output, ContentReleaseIdentifier $contentReleaseIdentifier): self + public static function fromConsoleOutput(ConsoleOutput $output, ContentReleaseIdentifier $contentReleaseIdentifier, StatisticsEventOutputInterface $statisticsEventOutput = new RedisStatisticsEventOutput()): self { - return new static($output->getOutput(), $contentReleaseIdentifier, null); + return new static($output->getOutput(), $contentReleaseIdentifier, $statisticsEventOutput, null); } - public static function fromSymfonyOutput(OutputInterface $output, ContentReleaseIdentifier $contentReleaseIdentifier): self + public static function fromSymfonyOutput(OutputInterface $output, ContentReleaseIdentifier $contentReleaseIdentifier, StatisticsEventOutputInterface $statisticsEventOutput = new RedisStatisticsEventOutput()): self { - return new static($output, $contentReleaseIdentifier, null); + return new static($output, $contentReleaseIdentifier, $statisticsEventOutput, null); } public function debug($message, array $additionalPayload = []) @@ -72,8 +78,13 @@ public function logException(\Exception $exception, string $message, array $addi $this->output->writeln($this->logPrefix . $message . "\n\n" . $exception->getMessage() . "\n\n" . $exception->getTraceAsString() . "\n\n" . json_encode($additionalPayload)); } + public function logStatisticsEvent(string $event, array $additionalPayload = []) + { + $this->statisticsEventOutput->writeEvent($this->contentReleaseIdentifier, $this->logPrefix, $event, $additionalPayload); + } + public function withRenderer(RendererIdentifier $rendererIdentifier): self { - return new ContentReleaseLogger($this->output, $this->contentReleaseIdentifier, $rendererIdentifier); + return new ContentReleaseLogger($this->output, $this->contentReleaseIdentifier, $this->statisticsEventOutput, $rendererIdentifier); } } diff --git a/Classes/Core/Infrastructure/RedisStatisticsEventOutput.php b/Classes/Core/Infrastructure/RedisStatisticsEventOutput.php new file mode 100644 index 0000000..b4b58cd --- /dev/null +++ b/Classes/Core/Infrastructure/RedisStatisticsEventOutput.php @@ -0,0 +1,27 @@ +redisClientManager->getPrimaryRedis()->rPush($this->redisKeyService->getRedisKeyForPostfix($contentReleaseIdentifier, 'statisticsEvents'), json_encode([ + 'event' => $event, + 'prefix' => $prefix, + 'additionalPayload' => $additionalPayload, + ])); + } +} diff --git a/Classes/Core/Infrastructure/StatisticsEventOutputInterface.php b/Classes/Core/Infrastructure/StatisticsEventOutputInterface.php new file mode 100644 index 0000000..fb0cd6b --- /dev/null +++ b/Classes/Core/Infrastructure/StatisticsEventOutputInterface.php @@ -0,0 +1,11 @@ + Date: Thu, 17 Jul 2025 11:32:36 +0200 Subject: [PATCH 08/10] FEATURE: Add command line command to read statistics events --- .../ContentReleaseEventsCommandController.php | 58 +++++++++ .../RedisStatisticsEventOutput.php | 13 +- .../RedisStatisticsEventService.php | 121 ++++++++++++++++++ 3 files changed, 181 insertions(+), 11 deletions(-) create mode 100644 Classes/Command/ContentReleaseEventsCommandController.php create mode 100644 Classes/Core/Infrastructure/RedisStatisticsEventService.php diff --git a/Classes/Command/ContentReleaseEventsCommandController.php b/Classes/Command/ContentReleaseEventsCommandController.php new file mode 100644 index 0000000..3bf2e2e --- /dev/null +++ b/Classes/Command/ContentReleaseEventsCommandController.php @@ -0,0 +1,58 @@ + explode('=', $s, 2), explode(',', $where)), 1, 0) : []; + $groupBy = $groupBy ? explode(',', $groupBy) : []; + + $this->output("Filters: \n"); + if($where) { + foreach ($where as $key=>$value) { + $this->output(" $key = \"$value\"\n"); + } + } else { + $this->output(" None \n"); + } + + $eventCounts = $this->redisStatisticsEventService->countEvents($contentReleaseIdentifier, $where, $groupBy); + $this->output->outputTable($eventCounts, array_merge(['count'], $groupBy)); + + $this->output("Total: %d\n", [array_sum(array_map(fn($e) => $e['count'], $eventCounts))]); + } +} diff --git a/Classes/Core/Infrastructure/RedisStatisticsEventOutput.php b/Classes/Core/Infrastructure/RedisStatisticsEventOutput.php index b4b58cd..ce8f2a0 100644 --- a/Classes/Core/Infrastructure/RedisStatisticsEventOutput.php +++ b/Classes/Core/Infrastructure/RedisStatisticsEventOutput.php @@ -4,24 +4,15 @@ namespace Flowpack\DecoupledContentStore\Core\Infrastructure; use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier; -use Flowpack\DecoupledContentStore\Core\RedisKeyService; use Neos\Flow\Annotations as Flow; class RedisStatisticsEventOutput implements StatisticsEventOutputInterface { - - #[Flow\Inject] - protected RedisClientManager $redisClientManager; - #[Flow\Inject] - protected RedisKeyService $redisKeyService; + protected RedisStatisticsEventService $redisStatisticsEventService; public function writeEvent(ContentReleaseIdentifier $contentReleaseIdentifier, string $prefix, string $event, array $additionalPayload): void { - $this->redisClientManager->getPrimaryRedis()->rPush($this->redisKeyService->getRedisKeyForPostfix($contentReleaseIdentifier, 'statisticsEvents'), json_encode([ - 'event' => $event, - 'prefix' => $prefix, - 'additionalPayload' => $additionalPayload, - ])); + $this->redisStatisticsEventService->addEvent($contentReleaseIdentifier, $prefix, $event, $additionalPayload); } } diff --git a/Classes/Core/Infrastructure/RedisStatisticsEventService.php b/Classes/Core/Infrastructure/RedisStatisticsEventService.php new file mode 100644 index 0000000..23e7ec8 --- /dev/null +++ b/Classes/Core/Infrastructure/RedisStatisticsEventService.php @@ -0,0 +1,121 @@ +redisClientManager->getPrimaryRedis()->rPush($this->redisKeyService->getRedisKeyForPostfix($contentReleaseIdentifier, 'statisticsEvents'), json_encode([ + 'event' => $event, + 'prefix' => $prefix, + 'additionalPayload' => $additionalPayload, + ])); + } + + /** + * @param ContentReleaseIdentifier $contentReleaseIdentifier + * @param array $where + * @param string[] $groupBy + * @return array<> + * @throws Exception + */ + public function countEvents( + ContentReleaseIdentifier $contentReleaseIdentifier, + array $where, + array $groupBy, + ): array + { + $redis = $this->redisClientManager->getPrimaryRedis(); + $key = $this->redisKeyService->getRedisKeyForPostfix($contentReleaseIdentifier, 'statisticsEvents'); + $chunkSize = 1000; + + $countedEvents = []; + + $listLength = $redis->lLen($key); + for ($start = 0; $start < $listLength; $start += $chunkSize) { + $events = $redis->lRange($key, $start, $start + $chunkSize - 1); + + foreach ($events as $eventJson) { + $event = $this->flatten(json_decode($eventJson, true)); + if($this->shouldCount($event, $where)) { + $group = $this->groupValues($event, $groupBy); + $eventKey = json_encode($group); + if (array_key_exists($eventKey, $countedEvents)) { + $countedEvents[$eventKey]['count'] += 1; + } else { + $countedEvents[$eventKey] = array_merge(['count' => 1], $group); + } + } + } + } + // throw away the keys and sort in _reverse_ order by count + usort($countedEvents, fn($a, $b) => $b['count'] - $a['count']); + return $countedEvents; + } + + /** + * @phpstan-type JSONArray array + *ยด + * @param JSONArray $array + * @return array + */ + private function flatten(array $array): array + { + $results = []; + + foreach ($array as $key => $value) { + if (is_array($value) && ! empty($value)) { + foreach ($this->flatten($value) as $subKey => $subValue) { + $results[$key . '.' . $subKey] = $subValue; + } + } else { + $results[$key] = $value; + } + } + + return $results; + } + + /** + * @param array $event + * @param array $where + * @return bool + */ + private function shouldCount(array $event, array $where): bool + { + foreach ($where as $key=>$value) { + if (!array_key_exists($key, $event) || $event[$key] !== $value) { + return false; + } + } + return true; + } + + /** + * @param array $event + * @param string[] $groupedBy + * @return array + */ + private function groupValues(array $event, array $groupedBy): array + { + $group = []; + foreach ($groupedBy as $path) { + $group[$path] = $event[$path] ?? null; + } + return $group; + } +} From a198cfd0e50364d3c71e1e57ab56236eef3f010f Mon Sep 17 00:00:00 2001 From: Robert Baruck Date: Tue, 2 Sep 2025 16:09:33 +0200 Subject: [PATCH 09/10] TASK: Add nodePathSegmentFilter to LimitEnumerator --- .../DocumentEnumerators/LimitEnumerator.php | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/Classes/NodeRendering/Extensibility/DocumentEnumerators/LimitEnumerator.php b/Classes/NodeRendering/Extensibility/DocumentEnumerators/LimitEnumerator.php index 6bae087..9899ca5 100644 --- a/Classes/NodeRendering/Extensibility/DocumentEnumerators/LimitEnumerator.php +++ b/Classes/NodeRendering/Extensibility/DocumentEnumerators/LimitEnumerator.php @@ -15,19 +15,29 @@ class LimitEnumerator implements DocumentEnumeratorInterface protected int $i = 0; private ?int $limit = null; private ?string $uriPathSegmentFilter = null; + private ?string $nodePathSegmentFilter = null; public function __construct( array $options = [] ) { $this->limit = $options['limit'] ?? null; $this->uriPathSegmentFilter = $options['uriPathSegmentFilter'] ?? null; + $this->nodePathSegmentFilter = $options['nodePathSegmentFilter'] ?? null; } public function enumerateDocumentNode(NodeInterface $documentNode): iterable { - if ($this->uriPathSegmentFilter !== null) { - if (str_contains($documentNode->getProperty('uriPathSegment'), $this->uriPathSegmentFilter) === false) { - return []; - } + if ( + $this->uriPathSegmentFilter !== null + && str_contains($documentNode->getProperty('uriPathSegment'), $this->uriPathSegmentFilter) === false + ) { + return []; + } + + if ( + $this->nodePathSegmentFilter !== null + && str_contains($documentNode->getPath(), $this->nodePathSegmentFilter) === false + ) { + return []; } // NOTE: Limiting must come LAST, after all other constraints have been evaluated (because we want From 48b535bbc054e40ef285e044bfb02a38742a4d4a Mon Sep 17 00:00:00 2001 From: Sebastian Kurfuerst Date: Mon, 9 Feb 2026 11:23:17 +0100 Subject: [PATCH 10/10] BUGFIX: do not crash if e.g. a page is called '404' in Redirects --- Classes/Transfer/ContentReleaseSynchronizer.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Classes/Transfer/ContentReleaseSynchronizer.php b/Classes/Transfer/ContentReleaseSynchronizer.php index d536476..b0eb704 100644 --- a/Classes/Transfer/ContentReleaseSynchronizer.php +++ b/Classes/Transfer/ContentReleaseSynchronizer.php @@ -134,7 +134,7 @@ protected function transferHashKeyIncrementally(\Redis $sourceRedis, \Redis $tar $numberOfBatches++; $targetPipeline = $targetRedis->pipeline(); // we don't care for the replies or for transactionality; so we use pipelining instead of MULTI foreach ($arr_keys as $hashKey => $hashValue) { - $targetPipeline->hSet($keyToTransfer, $hashKey, $hashValue); + $targetPipeline->hSet($keyToTransfer, (string)$hashKey, $hashValue); } $targetPipeline->exec(); }