Initial import
This commit is contained in:
@@ -0,0 +1,87 @@
|
||||
<?php
|
||||
|
||||
class DownloadJob extends Job {
|
||||
|
||||
public static function jobs() { return 3; }
|
||||
|
||||
public $from = null;
|
||||
public $to = null;
|
||||
|
||||
private $_pct = 0;
|
||||
|
||||
public function __construct($from) {
|
||||
$this->from = $from;
|
||||
parent::__construct("download");
|
||||
}
|
||||
|
||||
public function generate_to() {
|
||||
$filename = ROOT . "/download/file-" . time() . "-" . rand() . ".pdf";
|
||||
return $filename;
|
||||
}
|
||||
|
||||
public function run() {
|
||||
|
||||
|
||||
$this->to = $this->generate_to();
|
||||
$ch = curl_init();
|
||||
$this->status("Downloading: 0%");
|
||||
$fd = fopen($this->to, "w");
|
||||
|
||||
curl_setopt($ch, CURLOPT_URL, $this->from);
|
||||
curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36");
|
||||
curl_setopt($ch, CURLOPT_HEADER, 0);
|
||||
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 15);
|
||||
curl_setopt($ch, CURLOPT_PRIVATE, $this);
|
||||
curl_setopt($ch, CURLOPT_TIMEOUT, 3600);
|
||||
curl_setopt($ch, CURLOPT_FILETIME, true);
|
||||
curl_setopt($ch, CURLOPT_PROGRESSFUNCTION, [$this, 'download_progress']);
|
||||
curl_setopt($ch, CURLOPT_NOPROGRESS, false);
|
||||
curl_setopt($ch, CURLOPT_FILE, $fd);
|
||||
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
|
||||
|
||||
print("Running...\n");
|
||||
$r = curl_exec($ch);
|
||||
fclose($fd);
|
||||
|
||||
$file = new File($this->to);
|
||||
$sha256 = $file->hash();
|
||||
|
||||
$rev = Revision::find([["sha256", "=", $sha256]])->first();
|
||||
|
||||
if ($rev) {
|
||||
$this->status("Duplicate file");
|
||||
$this->fail();
|
||||
print("Finished (duplicate)\n");
|
||||
return;
|
||||
}
|
||||
|
||||
$u = new Upload();
|
||||
$u->filename = basename($this->from);
|
||||
$u->sha256 = $sha256;
|
||||
$u->source = $this->from;
|
||||
$u->ocr = "N";
|
||||
$u->ai = "N";
|
||||
$u->owner = $this->getOwner();
|
||||
$u->save();
|
||||
$file->rename(ROOT . "/uploads/" . $u->id . ".pdf");
|
||||
$j = new ImportGeminiJob($u->id);
|
||||
$j->queue($u->owner);
|
||||
$j = new ImportOCRJob($u->id);
|
||||
$j->queue($u->owner);
|
||||
|
||||
$this->status("Finished");
|
||||
print("Finished\n");
|
||||
$this->finish();
|
||||
}
|
||||
|
||||
|
||||
|
||||
function download_progress($ch, $download_size, $downloaded, $upload_size, $uploaded) {
|
||||
if ($download_size == 0) return;
|
||||
$pct = round($downloaded / $download_size * 100);
|
||||
if ($pct != $this->_pct) {
|
||||
$this->status("Downloading: " . $pct . "%");
|
||||
$this->_pct = $pct;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
<?php
|
||||
|
||||
class GeminiJob extends Job {
|
||||
|
||||
public static function jobs() { return 1; }
|
||||
|
||||
public $docid;
|
||||
public $move=false;
|
||||
|
||||
public function __construct($docid, $source="unknown") {
|
||||
parent::__construct($source);
|
||||
$this->docid = $docid;
|
||||
}
|
||||
|
||||
public function run() {
|
||||
|
||||
$pdf = false;
|
||||
|
||||
$subs = [
|
||||
"Installation and Operating Information",
|
||||
"Installation and Configuration",
|
||||
"Installing and Getting Started",
|
||||
"Installation/Operator's Manual",
|
||||
"Installation/Operator's Guide",
|
||||
"Programmer's Reference Guide",
|
||||
"Field Maintenance Print Set",
|
||||
"Illustrated Parts Breakdown",
|
||||
"Installation/Owner's Guide",
|
||||
"Installation Information",
|
||||
"Installation/User Guide",
|
||||
"User Documentation Kit",
|
||||
"Programmer Information",
|
||||
"Technical Description",
|
||||
"Operator Information",
|
||||
"Configuration Guide",
|
||||
"Upgrade Information",
|
||||
"Installation Manual",
|
||||
"Service Information",
|
||||
"Installation Guide",
|
||||
"Programming Manual",
|
||||
"Maintenance Manual",
|
||||
"Maintenance Guide",
|
||||
"Technical Summary",
|
||||
"Operator's Guide",
|
||||
"System Reference",
|
||||
"User Information",
|
||||
"Technical Manual",
|
||||
"Language Manual",
|
||||
"Service Manual",
|
||||
"Service Guide",
|
||||
"Read Me First",
|
||||
"Owner's Guide",
|
||||
"Release Notes",
|
||||
"Options Guide",
|
||||
"Users' Manual",
|
||||
"User's Manual",
|
||||
"HiTest Notes",
|
||||
"User's Guide",
|
||||
"Design Guide",
|
||||
"User Manual",
|
||||
"User Guide",
|
||||
|
||||
];
|
||||
|
||||
$doc = new Document($this->docid);
|
||||
|
||||
$minsize = 999999999999999;
|
||||
|
||||
foreach ($doc->revisions as $r) {
|
||||
$p = new PDF($r->path() . "/doc.pdf");
|
||||
if ($p->size() < $minsize) {
|
||||
$minsize = $p->size();
|
||||
$rev = $r;
|
||||
$pdf = $p;
|
||||
}
|
||||
}
|
||||
|
||||
if (!$pdf) {
|
||||
$this->fail("No files to upload");
|
||||
return;
|
||||
}
|
||||
|
||||
$prods = [];
|
||||
$db = DB::getInstance();
|
||||
$q = $db->query("select id, full_path from product");
|
||||
while ($r = $db->nextRecord($q)) {
|
||||
$prods[] = $r->id . "," . $r->full_path;
|
||||
}
|
||||
|
||||
if ($doc->is_incoming()) {
|
||||
$this->move = true;
|
||||
}
|
||||
|
||||
file_put_contents("/tmp/category-list.csv", implode("\n", $prods) . "\n");
|
||||
|
||||
$this->status("Processing document with Gemini");
|
||||
try {
|
||||
$gemini = new Gemini();
|
||||
// $gemini->verbose = true;
|
||||
|
||||
$p = new GeminiParameter("title", "object", "The title of the document (note: none of these parts should include the order number)", true);
|
||||
$p->add_child("main_title", "string", "The primary portion of the title, properly recapitalized to fit English grammar", true);
|
||||
$p->add_child("subtitle", "string", "Any portion of the title that could be used as a subtitle, such as 'user guide' or 'maintenance guide' etc. Remove this from the main title. Should not be the order number.");
|
||||
$p->add_child("subsubtitle", "string", "Any part of the title that could be extra information such as version number or language. Should not be the order number.");
|
||||
$gemini->add_parameter($p);
|
||||
|
||||
$p = new GeminiParameter("category", "integer", "The category the document best fits in from the provided list of categories in the file category-list.csv", true);
|
||||
$gemini->add_parameter($p);
|
||||
|
||||
$gemini->add_parameter(new GeminiParameter("order_number", "string", "The document order number, often in the format XX-YYYYY-ZZ where YYYYY is based on the product code and ZZ is the type of document. Never more than 15 characters long."));
|
||||
$gemini->add_parameter(new GeminiParameter("overview", "string", "A brief one-line overview of the document content in plain text format", true));
|
||||
$gemini->add_parameter(new GeminiParameter("summary", "string", "A longer summary of the document content in markdown format", true));
|
||||
$gemini->upload_callback([$this, "uploadcb"]);
|
||||
$gemini->process_callback([$this, "processcb"]);
|
||||
|
||||
$gemini->attach($pdf->basename(), $pdf);
|
||||
$gemini->attach("category-list.csv", new File("/tmp/category-list.csv"));
|
||||
|
||||
$lines = $gemini->geminiOverview();
|
||||
} catch (Exception $e) {
|
||||
$this->status($e->getMessage());
|
||||
if ($e->getMessage() == "The request timed out. Please try again.") {
|
||||
$this->retry("timeout");
|
||||
} else if ($e->getMessage() == "HTTP Error 0 requesting AI assistance") {
|
||||
$this->retry("error 0");
|
||||
} else if (str_starts_with($e->getMessage(), "You exceeded your current quota, please check your plan and billing details.")) {
|
||||
$this->retries++;
|
||||
if ($this->retries < 5) {
|
||||
$this->retry("quota exceeded");
|
||||
} else {
|
||||
$this->fail("Too many retries");
|
||||
}
|
||||
} else if ($e->getMessage() == "This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.") {
|
||||
$this->retry("overload");
|
||||
} else {
|
||||
$this->fail();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
$data = json_decode($lines, true);
|
||||
|
||||
if (!$data) {
|
||||
$this->fail("No response from AI");
|
||||
return;
|
||||
}
|
||||
|
||||
// Reload document since this took a while
|
||||
$doc = new Document($this->docid);
|
||||
|
||||
if (array_key_exists("title", $data)) {
|
||||
$doc->subtitle = "";
|
||||
$doc->subsubtitle = "";
|
||||
if (array_key_exists("main_title", $data["title"])) { $doc->title = $data['title']['main_title']; }
|
||||
if (array_key_exists("subtitle", $data["title"])) { $doc->subtitle = $data['title']['subtitle']; }
|
||||
if (array_key_exists("subsubtitle", $data["title"])) { $doc->subsubtitle = $data['title']['subsubtitle']; }
|
||||
if ($doc->subtitle != "") {
|
||||
$doc->title = str_replace(" " . $doc->subtitle, "", $doc->title);
|
||||
}
|
||||
}
|
||||
if (array_key_exists("order_number", $data)) { $doc->internal_id = substr($data['order_number'], 0, 20); }
|
||||
if (array_key_exists("summary", $data)) { $doc->overview = $data['summary']; }
|
||||
if (array_key_exists("overview", $data)) { $doc->oneliner = $data['overview']; }
|
||||
|
||||
|
||||
|
||||
$doc->save();
|
||||
|
||||
if (array_key_exists("category", $data)) {
|
||||
$cat = $data['category'];
|
||||
$q = $db->query("select * from product where id=:id", ["id" => $cat]);
|
||||
if ($r = $db->nextRecord($q)) {
|
||||
if ($this->move) {
|
||||
$q = $db->query("delete from docproduct where document=:id", ["id" => $doc->id]);
|
||||
}
|
||||
$edp = DocProduct::find([
|
||||
["product", "=", $r->id],
|
||||
["document", "=", $doc->id]
|
||||
])->first();
|
||||
|
||||
if (!$edp) {
|
||||
$dp = new DocProduct;
|
||||
$dp->document = $doc->id;
|
||||
$dp->product = $r->id;
|
||||
$dp->save();
|
||||
}
|
||||
} else {
|
||||
print("--- Incorrect category suggested: " . $cat . "\n");
|
||||
}
|
||||
}
|
||||
|
||||
$this->status("Finished");
|
||||
$this->finish();
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
public function uploadcb($percent) {
|
||||
$this->status("Uploading: " . $percent . "% complete");
|
||||
}
|
||||
|
||||
public function processcb($message) {
|
||||
$this->status($message);
|
||||
}
|
||||
|
||||
function cleanup($txt) {
|
||||
$txt = str_replace("Ø", "0", $txt);
|
||||
$txt = str_replace(".", " ", $txt);
|
||||
return $txt;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
<?php
|
||||
|
||||
class ImportGeminiJob extends Job {
|
||||
|
||||
public static function jobs() { return 1; }
|
||||
|
||||
public $upload_id;
|
||||
|
||||
public function __construct($upload_id) {
|
||||
parent::__construct("upload:" . $upload_id);
|
||||
$this->upload_id = $upload_id;
|
||||
}
|
||||
|
||||
public function run() {
|
||||
|
||||
Config::refresh();
|
||||
|
||||
$pdf = false;
|
||||
|
||||
$subs = [
|
||||
"Installation and Operating Information",
|
||||
"Installation and Configuration",
|
||||
"Installing and Getting Started",
|
||||
"Installation/Operator's Manual",
|
||||
"Installation/Operator's Guide",
|
||||
"Programmer's Reference Guide",
|
||||
"Field Maintenance Print Set",
|
||||
"Illustrated Parts Breakdown",
|
||||
"Installation/Owner's Guide",
|
||||
"Installation Information",
|
||||
"Installation/User Guide",
|
||||
"User Documentation Kit",
|
||||
"Programmer Information",
|
||||
"Technical Description",
|
||||
"Operator Information",
|
||||
"Configuration Guide",
|
||||
"Upgrade Information",
|
||||
"Installation Manual",
|
||||
"Service Information",
|
||||
"Installation Guide",
|
||||
"Programming Manual",
|
||||
"Maintenance Manual",
|
||||
"Maintenance Guide",
|
||||
"Technical Summary",
|
||||
"Operator's Guide",
|
||||
"System Reference",
|
||||
"User Information",
|
||||
"Technical Manual",
|
||||
"Language Manual",
|
||||
"Service Manual",
|
||||
"Service Guide",
|
||||
"Read Me First",
|
||||
"Owner's Guide",
|
||||
"Release Notes",
|
||||
"Options Guide",
|
||||
"Users' Manual",
|
||||
"User's Manual",
|
||||
"HiTest Notes",
|
||||
"User's Guide",
|
||||
"Design Guide",
|
||||
"User Manual",
|
||||
"User Guide",
|
||||
|
||||
];
|
||||
|
||||
$doc = new Upload($this->upload_id);
|
||||
$doc->ai = "P";
|
||||
$doc->save();
|
||||
$pdf = $doc->getPDF();
|
||||
|
||||
if (!$pdf) {
|
||||
$this->fail();
|
||||
$doc->ai = "F";
|
||||
$this->status("PDF File Missing");
|
||||
$doc->save();
|
||||
return;
|
||||
}
|
||||
|
||||
if ($pdf->size() > 50000000) {
|
||||
$this->status("File too large");
|
||||
$doc->ai = "F";
|
||||
$doc->save();
|
||||
$this->fail();
|
||||
return;
|
||||
}
|
||||
|
||||
$this->status("Processing " . $doc->filename . " with Gemini");
|
||||
try {
|
||||
$gemini = new Gemini();
|
||||
$gemini->upload_callback([$this, "uploadcb"]);
|
||||
$gemini->process_callback([$this, "processcb"]);
|
||||
$lines = $gemini->geminiOverview($pdf);
|
||||
} catch (Exception $e) {
|
||||
$mess = $e->getMessage();
|
||||
print($mess . "\n");
|
||||
|
||||
switch ($mess) {
|
||||
case "This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.":
|
||||
case "The request timed out. Please try again.":
|
||||
case "HTTP Error 0 requesting AI assistance":
|
||||
$this->retry();
|
||||
print("Retrying\n");
|
||||
break;
|
||||
default:
|
||||
$this->fail();
|
||||
$this->status(substr($mess, 0, 250));
|
||||
$doc->overview = $mess;
|
||||
$doc->ai = "F";
|
||||
$doc->save();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
$this->status("Postprocessing returned data");
|
||||
$ld = explode("\n", $lines);
|
||||
if (preg_match('/^\{(.*)\}$/', trim($ld[0]), $m)) {
|
||||
$title = trim($m[1]);
|
||||
|
||||
$title = str_replace("’", "'", $title);
|
||||
|
||||
$newsub = "";
|
||||
$newsubsub = "";
|
||||
|
||||
|
||||
$b = explode(":", $title);
|
||||
if (count($b) > 1) {
|
||||
$title = trim(array_shift($b));
|
||||
$newsub = trim(array_shift($b));
|
||||
$newsubsub = implode(": ", $b);
|
||||
}
|
||||
|
||||
if ($newsub == "") {
|
||||
foreach ($subs as $sub) {
|
||||
if (str_ends_with($title, $sub)) {
|
||||
$newsub = $sub;
|
||||
$title = substr($title, 0, 0 - (strlen($sub) + 1));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
$doc->title = $title;
|
||||
$doc->subtitle = $newsub;
|
||||
$doc->subsubtitle = $newsubsub;
|
||||
|
||||
array_shift($ld);
|
||||
if (trim($ld[0]) == "") {
|
||||
array_shift($ld);
|
||||
}
|
||||
}
|
||||
if (preg_match('/^\[(.*)\]$/', trim($ld[0]), $m)) {
|
||||
|
||||
$iid = IDMatch::find_docid($this->cleanup($m[1]));
|
||||
|
||||
if ($iid) {
|
||||
$doc->ai_docid = $iid[0];
|
||||
}
|
||||
array_shift($ld);
|
||||
}
|
||||
$doc->overview = implode("\n", $ld);
|
||||
$doc->ai = "Y";
|
||||
|
||||
$doc->save();
|
||||
$this->status("Finished");
|
||||
$this->finish();
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
public function uploadcb($percent) {
|
||||
$this->status("Uploading: " . $percent . "% complete");
|
||||
}
|
||||
|
||||
public function processcb($message) {
|
||||
$this->status($message);
|
||||
}
|
||||
|
||||
function cleanup($txt) {
|
||||
$txt = str_replace("Ø", "0", $txt);
|
||||
$txt = str_replace(".", " ", $txt);
|
||||
return $txt;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
<?php
|
||||
|
||||
class ImportOCRJob extends Job {
|
||||
|
||||
public static function jobs() { return 1; }
|
||||
|
||||
public $upload_id;
|
||||
|
||||
public function __construct($upload_id) {
|
||||
parent::__construct("upload:" . $upload_id);
|
||||
$this->upload_id = $upload_id;
|
||||
}
|
||||
|
||||
public function run() {
|
||||
|
||||
$pdf = false;
|
||||
|
||||
$doc = new Upload($this->upload_id);
|
||||
$doc->ocr = "P";
|
||||
$doc->save();
|
||||
$pdf = $doc->getPDF();
|
||||
|
||||
if (!$pdf) {
|
||||
$this->fail();
|
||||
$doc->ocr = "F";
|
||||
$this->status("PDF File Missing");
|
||||
$doc->save();
|
||||
return;
|
||||
}
|
||||
|
||||
$this->status("Processing " . $doc->filename . " with OCR");
|
||||
|
||||
$body = $pdf->text()->get_text();
|
||||
if ($body == "") {
|
||||
$this->status("Running OCR on PDF file");
|
||||
$ocr = new PDF($pdf->path() . "-ocr.pdf");
|
||||
$pdf->ocr($ocr);
|
||||
$ocr->rename($pdf->path(), true);
|
||||
$pdf = $doc->getPDF();
|
||||
$body = $pdf->text()->get_text();
|
||||
}
|
||||
|
||||
$doc->body = $body;
|
||||
|
||||
$docid = IDMatch::find_docid($doc->filename . " " . $body);
|
||||
if ($docid) {
|
||||
$doc->ocr_docid = $docid[0];
|
||||
}
|
||||
|
||||
$doc->ocr = "Y";
|
||||
$doc->save();
|
||||
|
||||
$this->status("Finished");
|
||||
$this->finish();
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
public function uploadcb($percent) {
|
||||
$this->status("Uploading: " . $percent . "% complete");
|
||||
}
|
||||
|
||||
public function processcb($message) {
|
||||
$this->status($message);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
<?php
|
||||
|
||||
class IndexJob extends Job {
|
||||
|
||||
public static function jobs() { return 5; }
|
||||
|
||||
public $revision;
|
||||
|
||||
|
||||
public function __construct($rev) {
|
||||
parent::__construct("revision:" . $rev);
|
||||
$this->revision = $rev;
|
||||
}
|
||||
|
||||
public function run() {
|
||||
|
||||
$db = DB::getInstance();
|
||||
|
||||
$rev = new Revision($this->revision);
|
||||
$rev->load("body");
|
||||
$body = $rev->body;
|
||||
|
||||
if ($body) {
|
||||
$body->counted = 'P';
|
||||
$body->save();
|
||||
|
||||
|
||||
$db->query("DELETE FROM word_index WHERE revision=:rev", ["rev" => $rev->id]);
|
||||
|
||||
$words = $body->count_words();
|
||||
|
||||
foreach ($words as $word=>$hits) {
|
||||
$db->query("INSERT INTO word_index (revision, word, hits) VALUES (:rev, :word, :hits)", [
|
||||
"rev" => $rev->id,
|
||||
"word" => substr($word, 0, 20),
|
||||
"hits" => $hits
|
||||
]);
|
||||
}
|
||||
$body->counted = 'Y';
|
||||
$body->save();
|
||||
$this->finish();
|
||||
} else {
|
||||
$this->fail();
|
||||
$this->status("No body to count");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
<?php
|
||||
|
||||
class OCRJob extends Job {
|
||||
|
||||
public static function jobs() { return 1; }
|
||||
public $revision;
|
||||
|
||||
public function __construct($rev) {
|
||||
parent::__construct("revision:" . $rev);
|
||||
$this->revision = $rev;
|
||||
}
|
||||
|
||||
public function run() {
|
||||
$ocr = OCR::find([["revision", "=", $this->revision]])->first();
|
||||
if (!$ocr) {
|
||||
$ocr = new OCR;
|
||||
$ocr->revision = $this->revision;
|
||||
$ocr->save();
|
||||
}
|
||||
|
||||
$this->status("Processing");
|
||||
|
||||
$rev = new Revision($this->revision);
|
||||
$pdf = $rev->getPDF();
|
||||
$text = $pdf->text()->get_text();
|
||||
|
||||
print("[" . $text . "]\n");
|
||||
|
||||
if ($text == "") {
|
||||
$this->status("Running OCR on PDF");
|
||||
$newpdf = new PDF($pdf->parent() . "/ocr.pdf");
|
||||
$pdf->ocr($newpdf);
|
||||
$text = $newpdf->text()->get_text();
|
||||
}
|
||||
|
||||
$ocr->body = $text;
|
||||
$ocr->save();
|
||||
$j = new IndexJob($this->revision);
|
||||
$j->queue();
|
||||
$this->status("Complete");
|
||||
$this->finish();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
<?php
|
||||
|
||||
class ProcessJob extends Job {
|
||||
|
||||
public static function jobs() { return 1; }
|
||||
|
||||
public $docid = 0;
|
||||
|
||||
public function __construct($docid) {
|
||||
parent::__construct("document:$docid");
|
||||
$this->docid = $docid;
|
||||
}
|
||||
|
||||
public function run() {
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
<?php
|
||||
|
||||
class RecompressJob extends Job {
|
||||
|
||||
public static function jobs() { return 1; }
|
||||
|
||||
public $RevisionID;
|
||||
|
||||
public function __construct($id) {
|
||||
parent::__construct("revision:" . $id);
|
||||
$this->RevisionID = $id;
|
||||
}
|
||||
public function run() {
|
||||
$this->status("Recompressing");
|
||||
$rev = new Revision($this->RevisionID);
|
||||
$crev = new Revision;
|
||||
|
||||
$crev->revid = $rev->revid;
|
||||
$crev->save();
|
||||
|
||||
$from = $rev->getPDF();
|
||||
$to = $crev->getPDF();
|
||||
|
||||
$to->parent()->mkdir();
|
||||
$from->recompress($to->path());
|
||||
|
||||
$crev->document = $rev->document;
|
||||
$crev->info = $to->info();
|
||||
$crev->sha256 = $to->hash();
|
||||
$crev->save();
|
||||
|
||||
$this->status("New revision " . $crev->id);
|
||||
$this->finish();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
<?php
|
||||
|
||||
class RedownloadJob extends Job {
|
||||
|
||||
public static function jobs() { return 1; }
|
||||
|
||||
public $from = null;
|
||||
public $to = null;
|
||||
public $revid = null;
|
||||
|
||||
private $_pct = 0;
|
||||
|
||||
public function __construct($revid, $from, $to) {
|
||||
$this->from = $from;
|
||||
$this->to = $to;
|
||||
$this->revid = $revid;
|
||||
parent::__construct("revision:" . $revid);
|
||||
}
|
||||
|
||||
public function run() {
|
||||
$ch = curl_init();
|
||||
$this->status("Downloading: 0%");
|
||||
$fd = fopen($this->to, "w");
|
||||
|
||||
curl_setopt($ch, CURLOPT_URL, $this->from);
|
||||
curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36");
|
||||
curl_setopt($ch, CURLOPT_HEADER, 0);
|
||||
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 15);
|
||||
curl_setopt($ch, CURLOPT_PRIVATE, $this);
|
||||
curl_setopt($ch, CURLOPT_TIMEOUT, 3600);
|
||||
curl_setopt($ch, CURLOPT_FILETIME, true);
|
||||
curl_setopt($ch, CURLOPT_PROGRESSFUNCTION, [$this, 'download_progress']);
|
||||
curl_setopt($ch, CURLOPT_NOPROGRESS, false);
|
||||
curl_setopt($ch, CURLOPT_FILE, $fd);
|
||||
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
|
||||
|
||||
print("Running...\n");
|
||||
$r = curl_exec($ch);
|
||||
fclose($fd);
|
||||
$sha = hash_file("sha256", $this->to);
|
||||
$rev = new Revision($this->revid);
|
||||
$rev->sha256 = $sha;
|
||||
$rev->save();
|
||||
print("Finished\n");
|
||||
$this->finish();
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
|
||||
function download_progress($ch, $download_size, $downloaded, $upload_size, $uploaded) {
|
||||
if ($download_size == 0) return;
|
||||
$pct = round($downloaded / $download_size * 100);
|
||||
if ($pct != $this->_pct) {
|
||||
$this->status("Downloading: " . $pct . "%");
|
||||
$this->_pct = $pct;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user