Все банально и просто, нужно парсить страницы сайта и делать импорт (апдейт) категорий и товаров, уже нереально намучался т.к. при выполнении скрипта дольше около 30-35 секунд его работа обрывается и получаю в броузере ошибку "Веб-страница недоступна" (ERR_CONNECTION_RESET). Помогите понять почему такое происходит, все PHP конфиги установлены верно:
max_execution_time = 3600
max_input_time = 3600
memory_limit = 512M
На всякий даже в скрипте устанавливаю лимиты:
ini_set("memory_limit","200M");
ini_set("max_execution_time",3600);
set_time_limit(3600);
Код контроллера:
<?php
class ControllerToolBrainupdate extends Controller {
private $error = array();
public function index() {
$this->document->setTitle("бла бла бла");
$this->load->model('tool/brainupdate');
$res = array();
if (($this->request->server['REQUEST_METHOD'] == 'POST') && ($this->validate())) {
if ($this->model_tool_brainupdate->autorization()) {
$res = $this->model_tool_brainupdate->updateCategories();
if($res['status']) {
$res = $this->model_tool_brainupdate->updateProducts();
$this->session->data['success'] = "бла бла бла!";
$this->redirect($this->url->link('tool/brainupdate', 'token=' . $this->session->data['token'], 'SSL'));
} else {
$this->error['warning'] = "бла бла бла";
}
} else {
$this->error['warning'] = "бла бла бла :(";
}
}
$this->data['result'] = $res;
$this->data['action'] = $this->url->link('tool/brainupdate', 'token=' . $this->session->data['token'], 'SSL');
if (isset($this->error['warning'])) {
$this->data['error_warning'] = $this->error['warning'];
} else {
$this->data['error_warning'] = '';
}
if (isset($this->session->data['success'])) {
$this->data['success'] = $this->session->data['success'];
unset($this->session->data['success']);
} else {
$this->data['success'] = '';
}
$this->data['breadcrumbs'] = array();
$this->data['breadcrumbs'][] = array(
'text' => $this->language->get('text_home'),
'href' => $this->url->link('common/home', 'token=' . $this->session->data['token'], 'SSL'),
'separator' => FALSE
);
$this->data['breadcrumbs'][] = array(
'text' => "Обновление цен и категорий с BRAIN",
'href' => $this->url->link('tool/brainupdate', 'token=' . $this->session->data['token'], 'SSL'),
'separator' => ' :: '
);
$this->template = 'tool/brainupdate.tpl';
$this->children = array(
'common/header',
'common/footer',
);
$this->response->setOutput($this->render());
}
private function validate() {
if (!$this->user->hasPermission('modify', 'tool/brainupdate')) {
$this->error['warning'] = $this->language->get('error_permission');
}
if (!$this->error) {
return TRUE;
} else {
return FALSE;
}
}
}
Модель
<?php
include 'simple_html_dom.php';
class ModelToolBrainupdate extends Model {
const COUNT_BY = 500;
const START_FROM = 0;
var $BRAIN_URL = "http://www.бла";
var $LOGIN = "бла";
var $PASSWORD = "бла";
private static $CATEGORIES = array();
private static $currentCategoryID = 0;
function autorization() {
$data = str_get_html($this->makeGetRequest($this->BRAIN_URL . "/index.php?id=startpage&user=" .
$this->LOGIN . "&pass=" .
$this->PASSWORD . "&submit.x=12&submit.y=13&logintype=login&pid=6", true));
return strcmp($data->find("#rTD #rt_4 .rBlD1 .rBlD3", 1) -> plaintext, $this->LOGIN) == 0;
}
function updateCategories() {
$database =& $this->db;
//update categories
$data = str_get_html($this->makeGetRequest($this->BRAIN_URL . "/?id=products_cat&category=first", false));
$categories = $data->find("#rt_3 .dtree script", 0) -> innertext;
$categories = substr($categories, strpos($categories, "d.add(0,-1,''); ") + strlen("d.add(0,-1,''); "));
$categories = trim(str_replace(" document.write(d);", "", $categories));
$categories = preg_replace("/d.add\((([\d]*),([\d]*),\'(.*?)\',\'(.*?)\')\);/i", "$1\n" , $categories);
$categories = preg_split("/\n/", $categories);
foreach ($categories as $category) {
$categoryID = trim(preg_replace("/([\d]*),([\d]*),\'(.*?)\',\'(.*?)\'/", "$1" , trim($category)));
$parentCategoryID = trim(preg_replace("/([\d]*),([\d]*),\'(.*?)\',\'(.*?)\'/", "$2" , trim($category)));
$categoryName = trim(preg_replace("/([\d]*),([\d]*),\'(.*?)\',\'(.*?)\'/", "$3" , trim($category)));
$categoryURL = trim(preg_replace("/([\d]*),([\d]*),\'(.*?)\',\'(.*?)\'/", "$4" , trim($category)));
$realCategoryID = preg_split("/\=/", $categoryURL);
$realCategoryID = $realCategoryID[count($realCategoryID) - 1];
if (empty($categoryID) || empty($categoryName)) {
continue;
}
self::$CATEGORIES[$realCategoryID] = array(
'category_id'=> $categoryID,
'parent_id' => $parentCategoryID,
'name' => $categoryName);
}
return $this->importCategoriesIntoDatabase($database, self::$CATEGORIES);
}
function updateProducts() {
ini_set("memory_limit","200M");
ini_set("max_execution_time",3600);
set_time_limit(3600);
$database =& $this->db;
$data = str_get_html($this->makeGetRequest($this->BRAIN_URL . "/index.php?3&backPID=3&begin_at=" . self::START_FROM, false));
$this->importProductsIntoDatabase($database, $this->getProductsList($data));
$end = $data->find("#informPaneTD #nextPrevCell a");
unset($data);
$end = explode("=", end($end) -> getAttribute ("href"));
$end = end($end);
//for ($i = 1; $i <= $end / self::COUNT_BY; $i++) {
for ($i = 1; $i <= 2; $i++) {
$data = str_get_html($this->makeGetRequest($this->BRAIN_URL . "/index.php?3&backPID=3&begin_at=" . (self::COUNT_BY * $i), false));
$this->importProductsIntoDatabase($database, $this->getProductsList($data));
//sleep(2);
}
return TRUE;
}
function getProductsList(&$data) {
$products = array();
$elements = $data->find("#informPaneTD #newsCell tr td.prT, #informPaneTD #newsCell tr td.prTl");
foreach ($elements as $element) {
if (strcmp($element -> attr["class"], "prT") == 0) {
$categoryID = explode("=",$element -> find("#n1 a", 0) -> getAttribute("href"));
self::$currentCategoryID = end($categoryID);
if(!array_key_exists (self::$currentCategoryID, self::$CATEGORIES)) {
self::$currentCategoryID = 0;
}
} else if (strcmp($element -> attr["class"], "prTl") == 0) {
if (self::$currentCategoryID == 0) {
continue;
} else {
$productID = explode("=", $element -> find("a", 0) -> getAttribute ("href"));
array_push($products, array(
'product_id'=> end($productID),
'name' => $element -> find("#n2", 0) -> plaintext,
'category_id' => self::$CATEGORIES[self::$currentCategoryID]["category_id"],
'quantity' => 100,
'price' => $element -> find("#n3", 0) -> plaintext,
'stock_status_id' => 7));
}
}
}
return $products;
}
private function importProductsIntoDatabase( &$database, $products )
{
$res = array('status' => false,
'prod_add' => 0, 'prod_update' => 0);
// find the default language id
$languageId = $languageId = '1';
// start transaction, remove products
$sql = "START TRANSACTION;\n";
$sql.= "UPDATE `".DB_PREFIX."product` SET `stock_status_id`=5, `quantity`=0, `status`=0;\n";
$this->multiquery( $database, $sql );
// generate and execute SQL for storing the products
foreach ($products as $product) {
$sql = "SELECT product_id FROM `".DB_PREFIX."product` WHERE `product_id`=\"".$product['product_id']."\";";
$result = $database->query( $sql );
if ($result->rows) {
$sql = "UPDATE `".DB_PREFIX."product` SET `status`=1, `stock_status_id`=".$product['stock_status_id'].", `quantity`=".$product['quantity'].", `price`=".trim($product['price'])." WHERE `product_id`=\"".$product['product_id']."\";";
$database->query($sql);
$res['prod_update']++;
} else {
$productId = $product['product_id'];
$productName = $database->escape($product['name']);
$categoryId = $product['category_id'];
$quantity = $product['quantity'];
$model = $database->escape($product['name']);
$price = trim($product['price']);
$stockStatusId = $product['stock_status_id'];
$dateAdded = date("y-m-d");
$dateModified = date("y-m-d H:i:s");
$dateAvailable = date("y-m-d H:i:s");
$productDescription = '';
$manufacturerId = '0';
$imageName = '';
$shipping = '1';
$points = '0';
$weight = '0';
$weightClassId = '0';
$status = '1';
$taxClassId = '0';
$viewed = '0';
$meta_description = '';
$length = '0.0';
$width = '0.0';
$height = '0.0';
$seo_title = '';
$seo_h1 = '';
$lengthClassId = '1';
$sku = '';
$upc = '';
$ean = '';
$jan = '';
$isbn = '';
$mpn = '';
$location = '';
$storeId = '0';
$subtract = '1';
$minimum = '1';
$meta_keywords = '';
$tags = '';
$sort_order = '1';
$sql = "INSERT INTO `".DB_PREFIX."product` (`product_id`,`quantity`,`sku`,`upc`,`ean`,`jan`,`isbn`,`mpn`,`location`,";
$sql .= "`stock_status_id`,`model`,`manufacturer_id`,`image`,`shipping`,`price`,`points`,`date_added`,`date_modified`,`date_available`,`weight`,`weight_class_id`,`status`,";
$sql .= "`tax_class_id`,`viewed`,`length`,`width`,`height`,`length_class_id`,`sort_order`,`subtract`,`minimum`) VALUES ";
$sql .= "($productId,$quantity,'$sku','$upc','$ean','$jan','$isbn','$mpn','$location',";
$sql .= "$stockStatusId,'$model',$manufacturerId,'$imageName',$shipping,$price,$points,";
$sql .= ($dateAdded=='NOW()') ? "$dateAdded," : "'$dateAdded',";
$sql .= ($dateModified=='NOW()') ? "$dateModified," : "'$dateModified',";
$sql .= ($dateAvailable=='NOW()') ? "$dateAvailable," : "'$dateAvailable',";
$sql .= "$weight,$weightClassId,$status,";
$sql .= "$taxClassId,$viewed,$length,$width,$height,'$lengthClassId','$sort_order','$subtract','$minimum');";
$sql2 = "INSERT INTO `".DB_PREFIX."product_description` (`product_id`,`language_id`,`name`,`description`,`meta_description`,`meta_keyword`,`tag`,`seo_title`,`seo_h1`) VALUES ";
$sql2 .= "($productId,$languageId,'$productName','$productDescription','$meta_description','$meta_keywords','$tags','$seo_title','$seo_h1');";
$sql3 = "INSERT INTO `".DB_PREFIX."product_to_category` (`product_id`,`category_id`) VALUES ($productId,$categoryId)";
$sql4 = "INSERT INTO `".DB_PREFIX."product_to_store` (`product_id`,`store_id`) VALUES ($productId,$storeId);";
$database->query($sql);
$database->query($sql2);
$database->query($sql3);
$database->query($sql4);
$res['prod_add']++;
}
}
// final commit
$database->query("COMMIT;");
$res['status'] = true;
return $res;
}
private function importCategoriesIntoDatabase( &$database, &$categories )
{
$res = array('status' => false,
'cat_add' => 0, 'cat_update' => 0);
// start transaction, remove categories
$sql = "START TRANSACTION;\n";
$sql.= "UPDATE `".DB_PREFIX."category` SET `status`=0;\n";
$this->multiquery( $database, $sql );
// generate and execute SQL for inserting the categories
foreach ($categories as $category) {
$sql = "SELECT category_id FROM `".DB_PREFIX."category` WHERE `category_id`=\"".$category['category_id']."\";";
$result = $database->query( $sql );
if ($result->rows) {
$sql = "UPDATE `".DB_PREFIX."category` SET `status`=1 WHERE `category_id`=\"".$category['category_id']."\";";
$database->query($sql);
$res['cat_update']++;
} else {
$categoryId = $category['category_id'];
$parentId = $category['parent_id'];
$name = $database->escape($category['name']);
$dateAdded = date("y-m-d H:i:s");
$dateModified = date("y-m-d H:i:s");
$languageId = '1';
$imageName = '';
$top = '0';
$columns = '1';
$sortOrder = '0';
$description = '';
$meta_description = '';
$meta_keywords = '';
$seo_title = '';
$seo_h1 = '';
$storeId = '0';
$status = '1';
$sql2 = "INSERT INTO `".DB_PREFIX."category` (`category_id`, `image`, `parent_id`, `top`, `column`, `sort_order`, `date_added`, `date_modified`, `status`) VALUES ";
$sql2 .= "( $categoryId, '$imageName', $parentId, $top, $columns, $sortOrder, ";
$sql2 .= ($dateAdded=='NOW()') ? "$dateAdded," : "'$dateAdded',";
$sql2 .= ($dateModified=='NOW()') ? "$dateModified," : "'$dateModified',";
$sql2 .= " $status);";
$database->query( $sql2 );
$sql3 = "INSERT INTO `".DB_PREFIX."category_description` (`category_id`, `language_id`, `name`, `description`, `meta_description`, `meta_keyword`, `seo_title`, `seo_h1`) VALUES ";
$sql3 .= "( $categoryId, $languageId, '$name', '$description', '$meta_description', '$meta_keywords', '$seo_title', '$seo_h1' );";
$database->query( $sql3 );
$sql4 = "INSERT INTO `".DB_PREFIX."category_to_store` (`category_id`,`store_id`) VALUES ($categoryId,$storeId);";
$database->query($sql4);
$res['cat_add']++;
}
}
// final commit
$database->query( "COMMIT;" );
$res['status'] = true;
return $res;
}
private function multiquery( &$database, $sql ) {
foreach (explode(";\n", $sql) as $sql) {
$sql = trim($sql);
if ($sql) {
$database->query($sql);
}
}
}
private function makeGetRequest($URL, $saveCookie) {
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $URL ); // отправляем на
curl_setopt($ch, CURLOPT_HEADER, 0); // пустые заголовки
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); // возвратить то что вернул сервер
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1); // следовать за редиректами
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);// таймаут
if ($saveCookie) {
curl_setopt($ch, CURLOPT_COOKIEJAR, $_SERVER['DOCUMENT_ROOT'] . '/cookie.txt'); // сохранять куки в файл
} else {
curl_setopt($ch, CURLOPT_COOKIEFILE, $_SERVER['DOCUMENT_ROOT'] . '/cookie.txt');
}
$html = iconv("windows-1251", "utf-8", curl_exec($ch));
curl_close($ch);
return $html;
}
}
при входе в цикл for ($i = 1; $i <= 2; $i++) происходит это разъединение, ни каких ошибок в логах нет, ни в системном логе openCart-а, ни в логах Апача
