1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
|
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\Db;
use Exception;
use Piwik\Common;
use Piwik\Config;
use Piwik\Container\StaticContainer;
use Piwik\Db;
use Piwik\Log;
use Piwik\SettingsServer;
class BatchInsert
{
/**
* Performs a batch insert into a specific table by iterating through the data
*
* NOTE: you should use tableInsertBatch() which will fallback to this function if LOAD DATA INFILE not available
*
* @param string $tableName PREFIXED table name! you must call Common::prefixTable() before passing the table name
* @param array $fields array of unquoted field names
* @param array $values array of data to be inserted
* @param bool $ignoreWhenDuplicate Ignore new rows that contain unique key values that duplicate old rows
*/
public static function tableInsertBatchIterate($tableName, $fields, $values, $ignoreWhenDuplicate = true)
{
$fieldList = '(' . join(',', $fields) . ')';
$ignore = $ignoreWhenDuplicate ? 'IGNORE' : '';
foreach ($values as $row) {
$query = "INSERT $ignore INTO " . $tableName . "
$fieldList
VALUES (" . Common::getSqlStringFieldsArray($row) . ")";
Db::query($query, $row);
}
}
/**
* Performs a batch insert into a specific table using either LOAD DATA INFILE or plain INSERTs,
* as a fallback. On MySQL, LOAD DATA INFILE is 20x faster than a series of plain INSERTs.
*
* @param string $tableName PREFIXED table name! you must call Common::prefixTable() before passing the table name
* @param array $fields array of unquoted field names
* @param array $values array of data to be inserted
* @param bool $throwException Whether to throw an exception that was caught while trying
* LOAD DATA INFILE, or not.
* @param string $charset The charset to use, defaults to utf8
* @throws Exception
* @return bool True if the bulk LOAD was used, false if we fallback to plain INSERTs
*/
public static function tableInsertBatch($tableName, $fields, $values, $throwException = false, $charset = 'utf8')
{
$loadDataInfileEnabled = Config::getInstance()->General['enable_load_data_infile'];
if ($loadDataInfileEnabled
&& Db::get()->hasBulkLoader()) {
$path = self::getBestPathForLoadData();
$filePath = $path . $tableName . '-' . Common::generateUniqId() . '.csv';
try {
$fileSpec = array(
'delim' => "\t",
'quote' => '"', // chr(34)
'escape' => '\\\\', // chr(92)
'escapespecial_cb' => function ($str) {
return str_replace(array(chr(92), chr(34)), array(chr(92) . chr(92), chr(92) . chr(34)), $str);
},
'eol' => "\r\n",
'null' => 'NULL',
'charset' => $charset
);
self::createCSVFile($filePath, $fileSpec, $values);
if (!is_readable($filePath)) {
throw new Exception("File $filePath could not be read.");
}
$rc = self::createTableFromCSVFile($tableName, $fields, $filePath, $fileSpec);
if ($rc) {
unlink($filePath);
return true;
}
} catch (Exception $e) {
if ($throwException) {
throw $e;
}
}
// if all else fails, fallback to a series of INSERTs
if (file_exists($filePath)) {
@unlink($filePath);
}
}
self::tableInsertBatchIterate($tableName, $fields, $values);
return false;
}
private static function getBestPathForLoadData()
{
try {
$path = Db::fetchOne('SELECT @@secure_file_priv'); // was introduced in 5.0.38
} catch (Exception $e) {
// we do not rethrow exception as an error is expected if MySQL is < 5.0.38
// in this case tableInsertBatch might still work
}
if (empty($path) || !@is_dir($path) || !@is_writable($path)) {
$path = StaticContainer::get('path.tmp') . '/assets/';
} elseif (!Common::stringEndsWith($path, '/')) {
$path .= '/';
}
return $path;
}
/**
* Batch insert into table from CSV (or other delimited) file.
*
* @param string $tableName Name of table
* @param array $fields Field names
* @param string $filePath Path name of a file.
* @param array $fileSpec File specifications (delimiter, line terminator, etc)
*
* @throws Exception
* @return bool True if successful; false otherwise
*/
public static function createTableFromCSVFile($tableName, $fields, $filePath, $fileSpec)
{
// Chroot environment: prefix the path with the absolute chroot path
$chrootPath = Config::getInstance()->General['absolute_chroot_path'];
if (!empty($chrootPath)) {
$filePath = $chrootPath . $filePath;
}
// On Windows, MySQL expects forward slashes as directory separators
if (SettingsServer::isWindows()) {
$filePath = str_replace('\\', '/', $filePath);
}
$query = "
'$filePath'
REPLACE
INTO TABLE
`" . $tableName . "`";
if (isset($fileSpec['charset'])) {
$query .= ' CHARACTER SET ' . $fileSpec['charset'];
}
$fieldList = '(' . join(',', $fields) . ')';
$query .= "
FIELDS TERMINATED BY
'" . $fileSpec['delim'] . "'
ENCLOSED BY
'" . $fileSpec['quote'] . "'
";
if (isset($fileSpec['escape'])) {
$query .= " ESCAPED BY '" . $fileSpec['escape'] . "'";
}
$query .= "
LINES TERMINATED BY
'" . $fileSpec['eol'] . "'
$fieldList
";
/*
* First attempt: assume web server and MySQL server are on the same machine;
* this requires that the db user have the FILE privilege; however, since this is
* a global privilege, it may not be granted due to security concerns
*/
$keywords = array('');
/*
* Second attempt: using the LOCAL keyword means the client reads the file and sends it to the server;
* the LOCAL keyword may trigger a known PHP PDO\MYSQL bug when MySQL not built with --enable-local-infile
* @see http://bugs.php.net/bug.php?id=54158
*/
$openBaseDir = ini_get('open_basedir');
$safeMode = ini_get('safe_mode');
if (empty($openBaseDir) && empty($safeMode)) {
// php 5.x - LOAD DATA LOCAL INFILE is disabled if open_basedir restrictions or safe_mode enabled
$keywords[] = 'LOCAL ';
}
$exceptions = array();
foreach ($keywords as $keyword) {
$queryStart = 'LOAD DATA ' . $keyword . 'INFILE ';
$sql = $queryStart . $query;
try {
$result = @Db::exec($sql);
if (empty($result) || $result < 0) {
continue;
}
return true;
} catch (Exception $e) {
$code = $e->getCode();
$message = $e->getMessage() . ($code ? "[$code]" : '');
$exceptions[] = "\n Try #" . (count($exceptions) + 1) . ': ' . $queryStart . ": " . $message;
}
}
if (count($exceptions)) {
$message = "LOAD DATA INFILE failed... Error was: " . implode(",", $exceptions);
Log::info($message);
throw new Exception($message);
}
return false;
}
/**
* Create CSV (or other delimited) files
*
* @param string $filePath filename to create
* @param array $fileSpec File specifications (delimiter, line terminator, etc)
* @param array $rows Array of array corresponding to rows of values
* @throws Exception if unable to create or write to file
*/
protected static function createCSVFile($filePath, $fileSpec, $rows)
{
// Set up CSV delimiters, quotes, etc
$delim = $fileSpec['delim'];
$quote = $fileSpec['quote'];
$eol = $fileSpec['eol'];
$null = $fileSpec['null'];
$escapespecial_cb = $fileSpec['escapespecial_cb'];
$fp = @fopen($filePath, 'wb');
if (!$fp) {
throw new Exception('Error creating the tmp file ' . $filePath . ', please check that the webserver has write permission to write this file.');
}
foreach ($rows as $row) {
$output = '';
foreach ($row as $value) {
if (!isset($value) || is_null($value) || $value === false) {
$output .= $null . $delim;
} else {
$output .= $quote . $escapespecial_cb($value) . $quote . $delim;
}
}
// Replace delim with eol
$output = substr_replace($output, $eol, -1);
$ret = fwrite($fp, $output);
if (!$ret) {
fclose($fp);
throw new Exception('Error writing to the tmp file ' . $filePath);
}
}
fclose($fp);
@chmod($filePath, 0777);
}
}
|