Skip to content

Commit 154a80d

Browse files
engcom-Echoengcom-Echo
authored
Merge branch '2.4-develop' into no-author/reports-sales
2 parents a18963b + de4dfb8 commit 154a80d

File tree

757 files changed

+10382
-5037
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

757 files changed

+10382
-5037
lines changed

‎app/code/Magento/Analytics/Model/ReportWriter.php‎

Lines changed: 57 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
use Magento\Analytics\ReportXml\DB\ReportValidator;
1111
use Magento\Framework\Filesystem\Directory\WriteInterface;
12+
use Magento\Framework\Filesystem\File\WriteInterface as FileWriteInterface;
1213

1314
/**
1415
* Writes reports in files in csv format
@@ -69,23 +70,7 @@ public function write(WriteInterface $directory, $path)
6970
continue;
7071
}
7172
}
72-
/** @var $providerObject */
73-
$providerObject = $this->providerFactory->create($provider['class']);
74-
$fileName = $provider['parameters'] ? $provider['parameters']['name'] : $provider['name'];
75-
$fileFullPath = $path . $fileName . '.csv';
76-
$fileData = $providerObject->getReport(...array_values($provider['parameters']));
77-
$stream = $directory->openFile($fileFullPath, 'w+');
78-
$stream->lock();
79-
$headers = [];
80-
foreach ($fileData as $row) {
81-
if (!$headers) {
82-
$headers = array_keys($row);
83-
$stream->writeCsv($headers);
84-
}
85-
$stream->writeCsv($this->prepareRow($row));
86-
}
87-
$stream->unlock();
88-
$stream->close();
73+
$this->prepareData($provider, $directory, $path);
8974
}
9075
if ($errorsList) {
9176
$errorStream = $directory->openFile($path . $this->errorsFileName, 'w+');
@@ -100,6 +85,61 @@ public function write(WriteInterface $directory, $path)
10085
return true;
10186
}
10287

88+
/**
89+
* Prepare report data
90+
*
91+
* @param array $provider
92+
* @param WriteInterface $directory
93+
* @param string $path
94+
* @return void
95+
* @throws \Magento\Framework\Exception\FileSystemException
96+
*/
97+
private function prepareData(array $provider, WriteInterface $directory, string $path)
98+
{
99+
/** @var $providerObject */
100+
$providerObject = $this->providerFactory->create($provider['class']);
101+
$fileName = $provider['parameters'] ? $provider['parameters']['name'] : $provider['name'];
102+
$fileFullPath = $path . $fileName . '.csv';
103+
104+
$stream = $directory->openFile($fileFullPath, 'w+');
105+
$stream->lock();
106+
107+
$headers = [];
108+
if ($providerObject instanceof \Magento\Analytics\ReportXml\BatchReportProviderInterface) {
109+
$fileData = $providerObject->getBatchReport(...array_values($provider['parameters']));
110+
do {
111+
$this->doWrite($fileData, $stream, $headers);
112+
$fileData = $providerObject->getBatchReport(...array_values($provider['parameters']));
113+
$fileData->rewind();
114+
} while ($fileData->valid());
115+
} else {
116+
$fileData = $providerObject->getReport(...array_values($provider['parameters']));
117+
$this->doWrite($fileData, $stream, $headers);
118+
}
119+
120+
$stream->unlock();
121+
$stream->close();
122+
}
123+
124+
/**
125+
* Write data to file
126+
*
127+
* @param \Traversable $fileData
128+
* @param FileWriteInterface $stream
129+
* @param array $headers
130+
* @return void
131+
*/
132+
private function doWrite(\Traversable $fileData, FileWriteInterface $stream, array $headers)
133+
{
134+
foreach ($fileData as $row) {
135+
if (!$headers) {
136+
$headers = array_keys($row);
137+
$stream->writeCsv($headers);
138+
}
139+
$stream->writeCsv($this->prepareRow($row));
140+
}
141+
}
142+
103143
/**
104144
* Replace wrong symbols in row
105145
*

‎app/code/Magento/Analytics/ReportXml/BatchReportProviderInterface.php‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
<?php
2+
/************************************************************************
3+
*
4+
* Copyright 2023 Adobe
5+
* All Rights Reserved.
6+
*
7+
* NOTICE: All information contained herein is, and remains
8+
* the property of Adobe and its suppliers, if any. The intellectual
9+
* and technical concepts contained herein are proprietary to Adobe
10+
* and its suppliers and are protected by all applicable intellectual
11+
* property laws, including trade secret and copyright laws.
12+
* Dissemination of this information or reproduction of this material
13+
* is strictly forbidden unless prior written permission is obtained
14+
* from Adobe.
15+
* ************************************************************************
16+
*/
17+
declare(strict_types=1);
18+
19+
namespace Magento\Analytics\ReportXml;
20+
21+
interface BatchReportProviderInterface
22+
{
23+
public const BATCH_SIZE = 10000;
24+
25+
/**
26+
* Returns one batch of the report data
27+
*
28+
* @param string $name
29+
* @return \IteratorIterator
30+
*/
31+
public function getBatchReport(string $name): \IteratorIterator;
32+
}

‎app/code/Magento/Analytics/ReportXml/Query.php‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,11 @@ class Query implements \JsonSerializable
3232
*/
3333
private $config;
3434

35+
/**
36+
* @var Select
37+
*/
38+
private $selectCount;
39+
3540
/**
3641
* Query constructor.
3742
*
@@ -94,4 +99,32 @@ public function jsonSerialize()
9499
'config' => $this->getConfig()
95100
];
96101
}
102+
103+
/**
104+
* Get SQL for get record count
105+
*
106+
* @return Select
107+
* @throws \Zend_Db_Select_Exception
108+
*/
109+
public function getSelectCountSql(): Select
110+
{
111+
if (!$this->selectCount) {
112+
$this->selectCount = clone $this->getSelect();
113+
$this->selectCount->reset(\Magento\Framework\DB\Select::ORDER);
114+
$this->selectCount->reset(\Magento\Framework\DB\Select::LIMIT_COUNT);
115+
$this->selectCount->reset(\Magento\Framework\DB\Select::LIMIT_OFFSET);
116+
$this->selectCount->reset(\Magento\Framework\DB\Select::COLUMNS);
117+
118+
$part = $this->getSelect()->getPart(\Magento\Framework\DB\Select::GROUP);
119+
if (!is_array($part) || !count($part)) {
120+
$this->selectCount->columns(new \Zend_Db_Expr('COUNT(*)'));
121+
return $this->selectCount;
122+
}
123+
124+
$this->selectCount->reset(\Magento\Framework\DB\Select::GROUP);
125+
$group = $this->getSelect()->getPart(\Magento\Framework\DB\Select::GROUP);
126+
$this->selectCount->columns(new \Zend_Db_Expr(("COUNT(DISTINCT ".implode(", ", $group).")")));
127+
}
128+
return $this->selectCount;
129+
}
97130
}

‎app/code/Magento/Analytics/ReportXml/ReportProvider.php‎

Lines changed: 45 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
/**
1212
* Providers for reports data
1313
*/
14-
class ReportProvider
14+
class ReportProvider implements BatchReportProviderInterface
1515
{
1616
/**
1717
* @var QueryFactory
@@ -28,6 +28,26 @@ class ReportProvider
2828
*/
2929
private $iteratorFactory;
3030

31+
/**
32+
* @var int
33+
*/
34+
private $currentPosition = 0;
35+
36+
/**
37+
* @var int
38+
*/
39+
private $countTotal = 0;
40+
41+
/**
42+
* @var \Magento\Framework\DB\Adapter\AdapterInterface
43+
*/
44+
private $connection;
45+
46+
/**
47+
* @var Query
48+
*/
49+
private $dataSelect;
50+
3151
/**
3252
* ReportProvider constructor.
3353
*
@@ -46,8 +66,7 @@ public function __construct(
4666
}
4767

4868
/**
49-
* Returns custom iterator name for report
50-
* Null for default
69+
* Returns custom iterator name for report. Null for default
5170
*
5271
* @param Query $query
5372
* @return string|null
@@ -71,4 +90,27 @@ public function getReport($name)
7190
$statement = $connection->query($query->getSelect());
7291
return $this->iteratorFactory->create($statement, $this->getIteratorName($query));
7392
}
93+
94+
/**
95+
* @inheritdoc
96+
*/
97+
public function getBatchReport(string $name): \IteratorIterator
98+
{
99+
if (!$this->dataSelect || $this->dataSelect->getConfig()['name'] !== $name) {
100+
$this->dataSelect = $this->queryFactory->create($name);
101+
$this->currentPosition = 0;
102+
$this->connection = $this->connectionFactory->getConnection($this->dataSelect->getConnectionName());
103+
$this->countTotal = $this->connection->fetchOne($this->dataSelect->getSelectCountSql());
104+
}
105+
106+
if ($this->currentPosition >= $this->countTotal) {
107+
return $this->iteratorFactory->create(new \ArrayIterator([]), $this->getIteratorName($this->dataSelect));
108+
}
109+
110+
$statement = $this->connection->query(
111+
$this->dataSelect->getSelect()->limit(self::BATCH_SIZE, $this->currentPosition)
112+
);
113+
$this->currentPosition += self::BATCH_SIZE;
114+
return $this->iteratorFactory->create($statement, $this->getIteratorName($this->dataSelect));
115+
}
74116
}

‎app/code/Magento/Analytics/Test/Unit/Model/ReportWriterTest.php‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,8 @@ protected function setUp(): void
107107
*/
108108
public function testWrite(array $configData, array $fileData, array $expectedFileData): void
109109
{
110+
$fileData = new \IteratorIterator(new \ArrayIterator($fileData));
111+
$emptyFileData = new \IteratorIterator(new \ArrayIterator([]));
110112
$errors = [];
111113
$this->configInterfaceMock
112114
->expects($this->once())
@@ -121,10 +123,10 @@ public function testWrite(array $configData, array $fileData, array $expectedFil
121123
$parameterName = isset(reset($configData)[0]['parameters']['name'])
122124
? reset($configData)[0]['parameters']['name']
123125
: '';
124-
$this->reportProviderMock->expects($this->once())
125-
->method('getReport')
126+
$this->reportProviderMock->expects($this->exactly(2))
127+
->method('getBatchReport')
126128
->with($parameterName ?: null)
127-
->willReturn($fileData);
129+
->willReturnOnConsecutiveCalls($fileData, $emptyFileData);
128130
$errorStreamMock = $this->getMockBuilder(
129131
FileWriteInterface::class
130132
)->getMockForAbstractClass();

‎app/code/Magento/Analytics/Test/Unit/ReportXml/QueryTest.php‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,4 +84,31 @@ public function testJsonSerialize()
8484

8585
$this->assertSame($expectedResult, $this->query->jsonSerialize());
8686
}
87+
88+
public function testGetSelectCountSql()
89+
{
90+
$resetParams = [
91+
Select::ORDER,
92+
Select::LIMIT_COUNT,
93+
Select::LIMIT_OFFSET,
94+
Select::COLUMNS
95+
];
96+
97+
$this->selectMock
98+
->expects($this->exactly(4))
99+
->method('reset')
100+
->willReturnCallback(
101+
function (string $value) use (&$resetParams) {
102+
$this->assertEquals(array_shift($resetParams), $value);
103+
}
104+
);
105+
106+
$this->selectMock
107+
->expects($this->once())
108+
->method('columns')
109+
->with(new \Zend_Db_Expr('COUNT(*)'))
110+
->willReturnSelf();
111+
112+
$this->assertEquals($this->selectMock, $this->query->getSelectCountSql());
113+
}
87114
}

‎app/code/Magento/Analytics/Test/Unit/ReportXml/ReportProviderTest.php‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,4 +155,55 @@ public function testGetReport()
155155
->willReturn($this->iteratorMock);
156156
$this->assertEquals($this->iteratorMock, $this->subject->getReport($reportName));
157157
}
158+
159+
/**
160+
* @return void
161+
*/
162+
public function testGetBatchReport()
163+
{
164+
$reportName = 'test_report';
165+
$connectionName = 'sales';
166+
167+
$this->queryFactoryMock->expects($this->once())
168+
->method('create')
169+
->with($reportName)
170+
->willReturn($this->queryMock);
171+
172+
$this->connectionFactoryMock->expects($this->once())
173+
->method('getConnection')
174+
->with($connectionName)
175+
->willReturn($this->connectionMock);
176+
177+
$this->queryMock->expects($this->once())
178+
->method('getConnectionName')
179+
->willReturn($connectionName);
180+
181+
$this->selectMock->expects($this->once())
182+
->method('limit')
183+
->with(ReportProvider::BATCH_SIZE, 0)
184+
->willReturn($this->selectMock);
185+
186+
$this->queryMock->expects($this->once())
187+
->method('getConfig')
188+
->willReturn(
189+
[
190+
'connection' => $connectionName
191+
]
192+
);
193+
194+
$this->connectionMock->expects($this->once())
195+
->method('query')
196+
->with($this->selectMock)
197+
->willReturn($this->statementMock);
198+
199+
$this->connectionMock->expects($this->once())
200+
->method('fetchOne')
201+
->willReturn(5);
202+
203+
$this->iteratorFactoryMock->expects($this->once())
204+
->method('create')
205+
->with($this->statementMock, null)
206+
->willReturn($this->iteratorMock);
207+
$this->assertEquals($this->iteratorMock, $this->subject->getBatchReport($reportName));
208+
}
158209
}

‎app/code/Magento/AwsS3/Test/Mftf/Helper/DummyMetadataCache.php‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -83,11 +83,4 @@ public function updateMetadata(string $path, array $objectMetadata, bool $persis
8383
public function storeFileNotExists(string $path): void
8484
{
8585
}
86-
87-
/**
88-
* @inheirtDoc
89-
*/
90-
public function storeDirNotExists(string $path): void
91-
{
92-
}
9386
}

0 commit comments

Comments
 (0)