Initial import

app/controllers/CoverController.php

@@ -0,0 +1,66 @@
+<?php
+
+class CoverController {
+
+	static function get_cover($_request, $id, $filename, $size = false) {
+
+		$rev = new Revision($id);
+		if ($rev === false) return "";
+
+		$path = $rev->path();
+
+		$pdf = new PDF($path . "/doc.pdf");
+		if (!$pdf->exists()) {
+			return [404, "Not Found"];
+		}
+
+		if (!$size) {
+			$imgpath = sprintf("%s/cover.jpg", $path);
+		} else {
+			$imgpath = sprintf("%s/cover-%d.jpg", $path, $size);
+		}
+
+		$file = new File($imgpath);
+
+		if ($file->exists()) {
+			$h = $file->hash();
+			$h = "\"$h\"";
+			if ($h == $_request->header("If-None-Match")) {
+				return [304, "Not Modified", [
+					"ETag" => $h,
+					"Cache-Control" => "public, max-age=86400, must-revalidate",
+				]];
+			}
+			return new Image($file);
+		}
+
+		$cover = new Image($path . "/cover.jpg");
+		if (!$cover->exists()) {
+			$pdf->extract_page(0, $cover->path());
+			$cover = new Image($path . "/cover.jpg");
+		}
+
+		if (!$cover->exists()) {
+			return [404, "Not Found"];
+		}
+
+		if ($size === false) $size = $cover->width();
+
+		if ($size >= $cover->width()) {
+			$h = $cover->hash();
+			$h = "\"$h\"";
+			if ($h == $_request->header("If-None-Match")) {
+				return [304, "Not Modified", [
+					"ETag" => $h,
+					"Cache-Control" => "public, max-age=86400, must-revalidate",
+				]];
+			}
+			return $cover;
+		}
+
+		$cover->scale($size);
+		$img = $cover->save($imgpath, "image/jpeg");
+		return $img;
+	}
+	
+}

app/controllers/DocumentController.php

@@ -0,0 +1,561 @@
+<?php
+
+class DocumentController {
+
+	public static function browse($pid = 0) {
+
+		$product = new Product($pid);
+
+		return blade("documents", ["product" => $product]);
+	}
+
+	public static function show($id) {
+		$doc = new Document($id);
+		$doc->load();
+
+		return blade("document", [
+			"doc" => $doc
+		]);
+	}
+
+	public static function api_get($id) {
+		return new Document($id);
+	}
+
+	public static function api_set($id) {
+		$doc = new Document($id);
+		if ($doc->valid()) {
+			foreach ($_POST as $k=>$v) {
+				if ($k != "products") {
+					$doc->$k = trim($v);
+				}
+			}
+			$doc->save();
+
+			$pids = explode(",", $_POST['products']);
+			
+			$dpl = DocProduct::find([["document", "=", $id]])->all();;
+
+			foreach ($dpl as $dp) {
+				$e = array_search($dp->product, $pids);
+				if ($e !== false) {
+					unset($pids[$e]);
+					continue;
+				}
+				$dp->delete();
+			}
+
+			foreach ($pids as $pid) {
+				$dp = new DocProduct;
+				$dp->document = $id;
+				$dp->product = $pid;
+				$dp->save();
+			}
+
+			$doc->cache_invalidate("products");
+
+		}
+		return $doc;
+	}
+
+	public static function api_get_products($id) {
+		$doc = new Document($id);
+		return $doc->products;
+	}
+
+	public static function api_move($id) {
+		$doc = new Document($id);
+		$from = $_POST['from'];
+		$to = $_POST['to'];
+
+		$dp = DocProduct([["document", "=", $id], ["product", "=", $from]])->first();
+		if ($dp) {
+			$dp->product = $to;
+			$dp->save();
+		}
+
+		$doc->cache_invalidate("products");
+
+		return [];
+	}
+
+	public static function del_docproduct($doc, $prod) {
+		$dp = DocProduct::find([["document", "=", $doc], ["product", "=", $prod]])->first();
+		if ($dp) {
+			$dp->delete();
+		}
+		return back();
+	}
+
+	public static function get_by_id($id) {
+		return Document::find([["internal_id", "=", trim($id)]])->first();
+	}
+
+	public static function api_merge($id) {
+		$from = new Document($id);
+		$to = new Document($_POST['to']);
+
+		if ($from->id == $to->id) return [];
+
+		if ($from && $to) {
+			$revs = Revision::find([["document", "=", $from->id]]);
+			for ($rev = $revs->first(); $rev = $revs->next(); ) {
+				$rev->document = $to->id;
+				$rev->save();
+			}
+
+			$from->delete();
+		}
+		return new Document($to->id); // Force refresh
+	}
+
+
+
+	public static function api_drag_drop($_request) {
+		$src_id = $_request->post("src_id");
+		$dst_id = $_request->post("dst_id");
+
+		$src_type = $_request->post("src_type");
+		$dst_type = $_request->post("dst_type");
+
+		$src_extra = $_request->post("src_extra");
+		$dst_extra = $_request->post("dst_extra");
+
+		$copy = $_request->post("copy") == "true";
+
+		if (($src_type == "document") && ($dst_type == "document")) {
+			// Merge src into dst
+			$src = new Document($src_id);
+			$dst = new Document($dst_id);
+
+			if ($src->id == $dst->id) return [];
+
+			foreach ($src->revisions as $r) {
+				$r->document = $dst->id;
+				$r->save();
+			}
+
+			$src->delete();
+			$dst->cache_invalidate("revisions");
+		}
+
+		if (($src_type == "document") && ($dst_type == "product")) {
+
+			if ($copy) {
+				$dp = new DocProduct();
+				$dp->document = $src_id;
+				$dp->product = $dst_id;
+				$dp->save();
+			} else {
+
+				// Move document into product
+				$dpl = DocProduct::find([["document", "=", $src_id], ["product", "=", $src_extra]])->all();
+				foreach ($dpl as $dp) {
+					$dp->product = $dst_id;
+					$dp->save();
+				}
+				$prod = new Product($src_extra);
+				$prod->cache_invalidate("documents");
+			}
+			$doc = new Document($src_id);
+			$doc->cache_invalidate("products");
+			$prod = new Product($dst_id);
+			$prod->cache_invalidate("documents");
+		}
+
+		if (($src_type == "product") && ($dst_type == "product")) {
+
+			if ($src_id == $dst_id) return [];
+			// Move product into product
+			$sp = new Product($src_id);
+			$dp = new Product($dst_id);
+			$sp->parent = $dp->id;
+			$sp->save();
+			$sp->cache_invalidate("parent");
+			$sp->cache_invalidate("children");
+			$dp->cache_invalidate("parent");
+			$dp->cache_invalidate("children");
+		}
+
+
+		return [200, ["didit" => "true"]];
+	}
+
+	public static function merge($id) {
+
+		$doc = new Document($id);
+
+		$proc = new Process("pdftk");
+
+		foreach ($doc->revisions as $rev) {
+			$proc->arg($rev->path() . "/doc.pdf");
+		}
+
+		$nr = new Revision;
+		$nr->document = $doc->id;
+		$nr->revno = "NEW";
+		$nr->save();
+
+
+		$out = new File($nr->path() . "/doc.pdf");
+		$out->parent()->mkdir();
+
+		$proc->arg("output");
+		$proc->arg((string)$out);
+		$r = $proc->execute();
+
+		if ($r != 0) {
+			print("<pre>");
+			print_r($proc->stderr());
+			print("</pre>");
+			exit(0);
+		}
+
+		return redirect("/document/" . $doc->id);
+	}
+
+	public static function create_overview($id, $move) {
+
+		$job = new GeminiJob($id, "document:$id");
+		$job->move = ($move == "move");
+		$jobid = $job->queue();
+		flash("success", "Job queued as ID " . $jobid);
+
+		return redirect("/document/" . $id);
+	}
+
+	public static function api_get_title_fragment($_request) {
+		$q = $_request->post("title");
+		$db = DB::getInstance();
+		$q1 = $db->query("
+			SELECT 
+				DISTINCT title 
+			FROM (
+				SELECT 
+					DISTINCT title 
+				FROM 
+					document 
+				WHERE 
+					title LIKE :s 
+				UNION SELECT 
+					DISTINCT subtitle AS title 
+				FROM 
+					document 
+				WHERE 
+					subtitle LIKE :s
+				UNION SELECT 
+					DISTINCT subsubtitle AS title 
+				FROM 
+					document 
+				WHERE subsubtitle LIKE :s
+			) AS DERIVED", ["s" => $q . "%"]);
+		$o = $db->all($q1);
+
+		if ($o->count() == 0) {
+			return [404, "Not Found"];
+		}
+		if ($o->count() != 1) {
+			return [413, "Content Too Large"];
+		}
+		return $o[0]->title;
+	}
+
+	public static function separate($id) {
+
+		$doc = new Document($id);
+		$firstprod = $doc->products[0];
+
+		$count = 0;
+
+		foreach ($doc->revisions as $rev) {
+			$count++;
+			if ($count == 1) {
+				continue;
+			}
+
+			$newdoc = $doc->duplicate();
+			$newdoc->subsubtitle .= " - $count";
+			$rev->document = $newdoc->id;
+			$rev->save();
+		}
+
+		
+		return redirect("/documents/" . $firstprod);
+	}
+
+	public static function api_get_metadata($id) {
+		return DocMeta::find([["document", "=", $id]])->orderBy("metadata")->all();
+	}
+	
+	public static function api_new_metadata($_request, $id) {
+		$doc = new Document($id);
+		$doc->set_metadata($_request->put("item_id"), "");
+		return DocMeta::find([["document", "=", $id]])->orderBy("metadata")->all();
+	}
+
+	public static function api_set_metadata($_request, $id, $metadata) {
+		$doc = new Document($id);
+		$doc->set_metadata($metadata, $_request->post('data'));
+		return DocMeta::find([["document", "=", $id]])->orderBy("metadata")->all();
+	}
+
+	public static function api_delete_metadata($id, $metadata) {
+		$doc = new Document($id);
+		$doc->remove_metadata($metadata);
+		return DocMeta::find([["document", "=", $id]])->orderBy("metadata")->all();
+	}
+
+	public static function api_available_metadata($id) {
+		$exist = DocMeta::find([["document", "=", $id]])->orderBy("metadata")->all();
+		$metas = MetaType::find()->all();
+
+
+		$out = new Collection();
+
+		foreach ($metas as $meta) {
+			$e = false;
+			foreach ($exist as $ex) {
+				if ($ex->metadata == $meta->id) {
+					$e = true;
+					break;
+				}
+			}
+
+			if (!$e) {
+				$cl = new stdClass;
+				$cl->key = $meta->id;
+				$cl->value = $meta->name;
+				$out->push($cl);
+			}
+		}
+
+		$out->sort("value");
+
+		return $out;
+	}
+
+	public static function delete_metadata($id, $metadata) {
+		$doc = new Document($id);
+		$doc->remove_metadata($metadata);
+		return redirect("/document/" . $id);
+	}
+
+	public static function api_guess_docid($id) {
+		$doc = new Document($id);
+		$docid = $doc->guess_docid();
+
+		return new Collection(["id" => $id, "docid" => $docid]);
+	}
+
+	public static function download_attachment($id, $filename) {
+		$doc = new Document($id);
+		$atts = $doc->get_attachments();
+		foreach ($atts as $f) {
+			if ($f->basename() == $filename) {
+				$f->set_header("Content-Disposition", "attachment; filename=\"$filename\"");
+
+				return $f;
+			}
+		}
+		return false;
+	}
+
+	public static function upload_attachment($id) {
+		$doc = new Document($id);
+		return blade("upload.attachment", ["doc" => $doc]);
+	}
+
+	public static function do_upload_attachment($_request, $id) {
+		$doc = new Document($id);
+		mkdir(ROOT . "/attachments/" . $doc->id, 0777);
+
+		$f = 0;
+		while (($file = $_request->file("file", $f)) !== false) {
+			$nf = new File($file['tmp_name']);
+			$nf->rename(ROOT . "/attachments/" . $doc->id . "/" . $file['name']);
+			$f++;
+		}
+		return redirect("/document/" . $doc->id);
+	}
+
+	public static function upload_document($_request) {
+		return blade("upload.document");
+	}
+
+	public static function do_upload_document($_request) {
+		$i = 0;
+		while ($f = $_request->file("file", $i)) {
+
+			$file = new File($f['tmp_name']);
+			$sha256 = $file->hash();
+
+			$rev = Revision::find([["sha256", "=", $sha256]])->first();
+
+			if ($rev) {
+				return redirect("/uploads/duplicate/" . $rev->id);
+			}
+
+			$u = new Upload();
+			$u->filename = $f['name'];
+			$u->sha256 = $sha256;
+			$u->ocr = "N";
+			$u->ai = "N";
+			$u->owner = get_user()->id;
+			$u->save();
+			copy($f['tmp_name'], ROOT . "/uploads/" . $u->id . ".pdf");
+			$j = new ImportGeminiJob($u->id);
+			$j->queue();
+			$j = new ImportOCRJob($u->id);
+			$j->queue();
+			$i++;
+		}
+		return redirect("/uploads");
+	}
+
+	public static function rerun_upload($id) {
+		$u = new Upload($id);
+		if ($u->ocr == "F") {
+			$u->ocr = "N";
+			$u->save();
+			$j = new ImportOCRJob($u->id);
+			$j->queue();
+		}
+
+		if ($u->ai == "F") {
+			$u->ai = "N";
+			$u->save();
+			$j = new ImportGeminiJob($u->id);
+			$j->queue();
+		}
+		return redirect("/uploads");
+	}
+
+	public static function rerun_all() {
+		$ai = Upload::find([["ai", "=", "F"], ["owner", "=", get_user()->id]])->all();
+		foreach ($ai as $u) {
+			$u->ai = "N";
+			$u->save();
+			$j = new ImportGeminiJob($u->id);
+			$j->queue();
+		}
+		$ocr = Upload::find([["ocr", "=", "F"], ["owner", "=", get_user()->id]])->all();
+		foreach ($ocr as $u) {
+			$u->ocr = "N";
+			$u->save();
+			$j = new ImportOCRJob($u->id);
+			$j->queue();
+		}
+		return redirect("/uploads");
+	}
+
+	public static function uploads($_request) {
+		return DocumentController::uploads_page($_request, 0);
+	}
+
+	public static function uploads_page($_request, $page) {
+		$uploads = Upload::find([["owner", "=", get_user()->id]])->orderBy("id")->limit($page * 20, 20)->all();
+		return blade("upload.list", ["uploads" => $uploads, "page" => $page]);
+	}
+
+	public static function discard_upload($_request, $id) {
+		$u = new Upload($id);
+		if ($u->owner != get_user()->id) {
+			return [403, "Not yours, punk!"];
+		}
+
+		DB::getInstance()->query("delete from job where source=:source", ["source" => "upload:" . $u->id]);
+
+		unlink($u->getPDF());
+		$u->delete();
+		return redirect("/uploads");
+	}
+
+	public static function import($_request, $id, $source) {
+		$u = new Upload($id);
+		if ($u->owner != get_user()->id) {
+			return [403, "Not yours, punk!"];
+		}
+
+		$doc = false;
+
+		if ($source == "ai") {
+			$doc = Document::find_by_docid($u->ai_docid);
+		} else if ($source == "ocr") {
+			$doc = Document::find_by_docid($u->ocr_docid);
+		}
+
+		if (!$doc) {
+			$doc = new Document;
+			if ($source == "ai") {
+				$doc->internal_id = $u->ai_docid;
+			} else if ($source == "ocr") {
+				$doc->internal_id = $u->ocr_docid;
+			} else {
+				$doc->internal_id = "XX-XXXXX-XX";
+			}
+			$doc->title = $u->title;
+			$doc->subtitle = $u->subtitle;
+			$doc->subsubtitle = $u->subsubtitle;
+			$doc->overview = $u->overview;
+			$doc->save();
+
+			$incoming = Product::find([["title", "=", "Incoming"]])->first();
+			$dp = new DocProduct();
+			$dp->document = $doc->id;
+			$dp->product = $incoming->id;
+			$dp->save();
+		}
+
+
+		$rev = new Revision();
+		$rev->document = $doc->id;
+		$rev->revid="";
+		$rev->sha256 = $u->sha256;
+		$rev->origtitle = $u->source;
+		$rev->save();
+
+
+		$ocr = new OCR();
+		$ocr->revision = $rev->id;
+		$ocr->body = $u->body;
+		$ocr->save();
+		
+        $out = new File($rev->path() . "/doc.pdf");
+        $out->parent()->mkdir();
+
+		$pdf = $u->getPDF();
+		$pdf->rename($out->path());
+
+		$db = DB::getInstance();
+		$db->query("DELETE FROM job WHERE source=:source", ["source" => "upload:" . $u->id]);
+		$u->delete();
+
+		$rev->info = $rev->getPDF()->info();
+		$rev->save();
+
+		$j = new IndexJob($rev->id);
+		$j->queue();
+
+
+		//return redirect("/uploads");
+		return redirect("/document/" . $doc->id);
+		
+	}
+
+    public static function delete_upload($_request, $id) {
+        $u = new Upload($id);
+        if ($u->owner != get_user()->id) {
+            return [403, "Not yours, punk!"];
+        }
+	}
+
+	public static function upload_duplicate($id) {
+		$r = new Revision($id);
+		return blade("upload.duplicate", ["rev" => $r]);
+	}
+
+	public static function view_upload($id) {
+		$u = new Upload($id);
+		return $u->getPDF();
+	}
+}

app/controllers/DownloadController.php

@@ -0,0 +1,31 @@
+<?php
+
+class DownloadController {
+	public static function get_download($id) {
+		return new DownloadJob($id);
+	}
+
+	public static function start_download() {
+	}
+
+	public static function api_downloads() {
+		$s = DownloadJob::find([["processed", "=", 0], ["owner", "=", get_user()->id]])->orderBy("queued")->limit(20);
+		return $s->all();
+	}
+
+	public static function api_add_download($_request) {
+		$url = $_request->put("url");
+		$d = new DownloadJob;
+		$d->url = $url;
+		$d->queued = time();
+		$d->started = 0;
+		$d->finished = 0;
+		$d->processed = 0;
+		$d->owner = get_user()->id;
+        $d->file = sprintf("download/file-%08X-%08X", rand(), time());
+
+		$d->save();
+		return DownloadController::api_downloads();
+	}
+
+}

app/controllers/FavouriteController.php

@@ -0,0 +1,33 @@
+<?php
+
+class FavouriteController {
+
+	public static function user_set_favourite($doc) {
+		$user = get_user();
+		if ($user) {
+			$user->set_favourite($doc);
+		}
+		return redirect("/document/" . $doc);
+	}
+
+	public static function user_unset_favourite($doc) {
+		$user = get_user();
+		if ($user) {
+			$user->unset_favourite($doc);
+		}
+		return redirect("/document/" . $doc);
+	}
+
+	public static function favourites() {
+		$user = get_user();
+		if (!$user) return [403, "Huh... yeah right"];
+		$f = $user->favourites();
+		$docs = new Collection;
+		foreach ($f as $fav) {
+			$doc = new Document($fav->document);
+			$docs->add($doc);
+		}
+		return blade("/favourites", ["docs" => $docs]);
+	}
+
+}

app/controllers/HomeController.php

@@ -0,0 +1,7 @@
+<?php
+
+class HomeController {
+	static function index() {
+		return blade("index");
+	}
+}

app/controllers/ImportController.php

@@ -0,0 +1,78 @@
+<?php
+
+class ImportController {
+	public static function downloads() {
+		return blade("downloads");
+	}
+
+	public static function imports() {
+		return blade("imports");
+	}
+
+	public static function api_imports() {
+		$is = ProcessJob::find([["imported", "=", 0]])->orderBy("queued")->limit(50)->all();
+		foreach ($is as $i) {
+			$i->load("revision");
+		}
+		return $is;
+	}
+
+	public static function api_delete_import($id) {
+		$d = new ProcessJob($id);
+		if ($d->valid()) {
+			$d->delete();
+		}
+		return ImportController::api_imports();
+	}
+
+	public static function api_set_import($_request, $id) {
+		$d = new ProcessJob($id);
+		if ($d->valid()) {
+			foreach ($_POST as $k=>$v) {
+				$d->$k = $v;
+			}
+			$d->save();
+		}
+		return ImportController::api_imports();
+	}
+
+	public static function api_add_document($_request, $id) {
+
+		$job = new ProcessJob($id);
+
+		$doc = Document::find([["internal_id", "=", $_request->put('internal_id')]])->first();
+		if (!$doc) {
+			$doc = new Document;
+			$doc->internal_id = trim($_request->put('internal_id'));
+			$doc->title = trim($_request->put('title'));
+			$doc->subtitle = trim($_request->put('subtitle'));
+			$doc->subsubtitle = trim($_request->put('subsubtitle'));
+			$doc->overview = trim($_request->put('overview'));
+			$doc->owner = get_user()->id;
+			$doc->save();
+			
+			$prods = explode(",", $_request->put('products'));
+			foreach ($prods as $product) {
+				$dp = new DocProduct;
+				$dp->document = $doc->id;
+				$dp->product = $product;
+				$dp->save();
+			}
+		}
+
+		$job->document = $doc->id;
+
+		$rev = new Revision($job->revision);
+		$rev->document = $doc->id;
+		$rev->revno = trim($_request->put('revno'));
+		$rev->month = $_request->put('month');
+		$rev->year = $_request->put('year');
+		$rev->owner = get_user()->id;
+		$rev->save();
+
+		$job->imported = time();
+		$job->save();
+
+		return ImportController::api_imports();
+	}
+}

app/controllers/JobController.php

@@ -0,0 +1,23 @@
+<?php
+
+class JobController {
+	public static function api_get_jobs($source) {
+		$db = DB::getInstance();
+		$q = $db->query("select * from job where source=:source order by queued", ["source" => $source]);
+		
+		$c = new Collection();
+
+		while ($r = $db->nextRecord($q)) {
+			$c->push($r);
+		}
+		return $c;
+	}
+
+	public static function api_delete_job($id) {
+		$db = DB::getInstance();
+		$q = $db->query("select * from job where id=:id", ["id" => $id]);
+		$r = $db->nextRecord($q);
+		$q = $db->query("delete from job where id=:id", ["id" => $id]);
+		return JobController::api_get_jobs($r->source);
+	}
+}

app/controllers/PDFController.php

@@ -0,0 +1,57 @@
+<?php
+
+class PDFController {
+	public static function download($_request, $id, $type, $filename) {
+
+		$rev = new Revision($id);
+		$path = $rev->path();
+
+		$rev->downloads++;
+		$rev->last_download = time();
+		$rev->save();
+
+		$disp = "attachment";
+		$mime = "application/octet-stream";
+
+		$file = null;
+
+		switch ($type) {
+			case "view":
+				$file = new PDF($path . "/doc.pdf");
+				break;
+			case "download":
+				$file = new PDF($path . "/doc.pdf");
+				$file->force_download();
+				break;
+			case "viewocr":
+				$file = new PDF($path . "/ocr.pdf");
+				break;
+			case "downloadocr":
+				$file = new PDF($path . "/ocr.pdf");
+				$file->force_download();
+				break;
+		}
+		$file->fake_filename($filename);
+
+		if ($file == null) {
+			return [404, "Not Found 1"];
+		}
+
+		if (!$file->exists()) {
+			return [404, "Not Found 2"];
+		}
+
+		$file->set_header("link", "<" . Config::get("URL") . "/pdf/" . $id . "/download/" . $filename  . '>; rel="canonical"');
+
+		return $file;
+	}
+
+	public static function get_page($id, $page) {
+		$rev = new Revision($id);
+		$page = $rev->get_page($page, 150);
+		if ($page) {
+			return $page;
+		}	
+		return [404, "Not Found"];
+	}
+}

app/controllers/ProductController.php

@@ -0,0 +1,164 @@
+<?php
+
+class ProductController {
+	public static function api_get_list($list) {
+		$db = DB::getInstance();
+		$pids = explode(",", $list);
+		$out = new Collection();
+		foreach ($pids as $pid) {
+			$q = $db->query("select * from product where id=:id", ["id" => $pid]);
+			if ($r = $db->nextRecord($q)) {
+				$out->push($r);
+			}
+		}
+		return $out;
+	}
+
+	public static function api_get_mru() {
+		$out = new Collection;
+		$mru = Session::get("mru");
+		if (!$mru) {
+			$mru = [];
+		}
+		foreach ($mru as $p) {
+			$prod = new Product($p);
+			if ($prod) {
+				$out->push($prod);
+			}
+		}
+		return $out;
+	}
+
+	public static function api_add_mru($id) {
+		$mru = Session::get("mru");
+		if (!$mru) {
+			$mru = [];
+		}
+		array_unshift($mru, $id);
+		$mru = array_unique($mru);
+		while (count($mru) > 5) {
+			array_pop($mru);
+		}
+		Session::set("mru", $mru);
+		return ProductController::api_get_mru();
+	}
+
+	public static function api_search() {
+
+		return Product::find([["full_path", "like", "%" . $_POST['search'] . "%"]])->orderBy("full_path")->all();
+	}
+
+	public static function api_add_child($_request, $id) {
+		$p = new Product;
+		$p->parent = $id;
+		$p->title = $_request->put("title");
+		$p->save();
+		return $p;
+	}
+
+	public static function api_set($id) {
+		$p = new Product($id);
+
+		$data = [];
+		foreach ($_POST as $k=>$v) {
+			$p->$k = $v;
+		}
+		$p->save();
+
+		return $p;
+	}
+
+	public static function api_move($id) {
+		$p = new Product($id);
+		$p->parent = $_POST['to'];
+		$p->save();
+		return $p;
+	}
+
+	public static function api_delete($id) {
+		$product = new Product($id);
+		if ($product->valid()) {
+			$laf = Product::find([["title", "=", "Lost and Found"]])->first();
+
+			DB::getInstance()->query("delete from docproduct where product=:pid", ["pid" => $product->id]);
+
+			foreach ($product->children as $c) {
+				$c->parent = $laf;
+				$c->save();
+			}
+			$product->load("parent");
+			$par = $product->parent;
+			$product->delete();
+			return $par;
+		}
+		return [];
+	}
+
+	public static function api_empty_trash($id) {
+		$prod = new Product($id);
+		$docs = $prod->documents;
+
+		foreach ($docs as $doc) {
+			$revs = $doc->revisions;
+			foreach ($revs as $rev) {
+				$rev->delete();
+			}
+
+			$dpl = DocProduct::find([["document", "=", $doc->id]])->all();
+			foreach ($dpl as $dp) {
+				$dp->delete();
+			}
+
+			$doc->delete();
+		}
+
+		$prod->invalidate("documents");
+
+
+		return [];
+
+	}
+
+    public static function api_available_metadata($id) {
+		$prod = new Product($id);
+        $exist = $prod->meta();
+        $metas = MetaType::find()->all();
+
+        $out = new Collection();
+
+        foreach ($metas as $meta) {
+			if (!in_array($meta->id, $exist)) {
+                $cl = new stdClass;
+                $cl->key = $meta->id;
+                $cl->value = $meta->name;
+                $out->push($cl);
+            }
+        }
+
+        $out->sort("value");
+
+        return $out;
+    }
+
+	public static function api_add_metadata($_request, $id) {
+		$prod = new Product($id);
+		$prod->add_meta($_request->put('item_id'));
+		return Collection::from_array($prod->meta());
+	}
+
+	public static function api_gemini_all($id, $move) {
+		$prod = new Product($id);
+
+		$c = new Collection();
+
+		foreach ($prod->documents as $doc) {
+			if ($prod->oneliner == "") {
+				$job = new GeminiJob($doc->id, "document:" . $doc->id);
+				$job->move = ($move == "move");
+				$jobid = $job->queue();
+				$c->push(["document" => $doc->id, "job" => $jobid]);
+			}
+		}
+		return $c;
+	}
+}

app/controllers/RevisionController.php

@@ -0,0 +1,107 @@
+<?php
+
+class RevisionController {
+
+	public static function show($id) {
+
+		$rev = new Revision($id);
+		$rev->load("document");
+
+
+		if (!@$rev->info->Pages) {
+			$rev->info = $rev->getPDF()->info();
+			$rev->save();
+		}
+
+		return blade("revision", ["rev" => $rev]);
+	}
+
+	public static function api_set($id) {
+		$r = new Revision($id);
+		if ($r !== false) {
+			foreach ($_POST as $k=>$v) {
+				$r->$k = trim($v);
+			}
+			$r->save();
+		}
+
+		return $r;
+	}
+
+	public static function delete($id) {
+		$r = new Revision($id);
+		$r->load("document");
+		$doc = $r->document;
+		$r->delete();
+		return redirect("/document/" . $doc->id);
+	}
+
+	public static function redownload($id) {
+		$r = new Revision($id);
+		$j = new RedownloadJob($r->id, $r->origtitle, $r->path() . "/doc.pdf");
+		$jobid = $j->queue();
+        flash("success", "Job queued as ID " . $jobid);
+        return redirect("/revision/" . $id);
+	}
+
+	public static function purge($id) {
+		$r = new Revision($id);
+		$r->purge();
+        return redirect("/revision/" . $id);
+	}
+
+	public static function recompress($id) {
+		$r = new Revision($id);
+		$j = new RecompressJob($r->id);
+		$j->queue();
+		flash("info", "Recompression job queued");
+        return redirect("/revision/" . $id);
+	}
+		
+	public static function ocr($id) {
+		$j = new OCRJob($id);
+		$j->queue();
+		return redirect("/revision/" . $id);
+	}
+
+	public static function rate_revision($id, $rating) {
+		$user = get_user();
+		if (!$user) {
+			return new Revision($id);
+		}
+		$db = DB::getInstance();
+		$db->query("INSERT INTO revision_rating (owner,revision,rating) values (:owner,:revision,:rating) ON DUPLICATE KEY UPDATE rating=:rating", [
+			"owner" => $user->id,
+			"revision" => $id,
+			"rating" => $rating
+		]);
+		$db->query("UPDATE revision SET rating=(SELECT AVG(rating) FROM revision_rating WHERE revision=:revision) WHERE id=:revision", ["revision" => $id]);
+
+		return new Revision($id);
+
+	}
+
+	public static function split_revision($id) {
+		$r = new Revision($id);
+		$r->load("document");
+		$doc = new Document();
+		$doc->title = $r->document->title;
+		$doc->subtitle = $r->document->subtitle;
+		$doc->subsubtitle = $r->document->subsubtitle;
+		$doc->overview = $r->document->overview;
+		$doc->oneliner = $r->document->oneliner;
+		$doc->internal_id = $r->document->internal_id;
+		$doc->save();
+		foreach ($r->document->products as $prod) {
+			$dp = new DocProduct();
+			$dp->document = $doc->id;
+			$dp->product = $prod->id;
+			$dp->save();
+		}
+		$r->document = $doc->id;
+		$r->save();
+
+		return redirect("/document/" . $doc->id);
+	}
+
+}

app/controllers/SearchController.php

@@ -0,0 +1,76 @@
+<?php
+
+class SearchController {
+	static public function api_title_search() {
+		$out = new Collection;
+
+		$q = DB::getInstance()->query("
+			select 
+				id, internal_id, title, subtitle, subsubtitle,
+				match (internal_id, title, subtitle, subsubtitle, overview)
+				against (:search in boolean mode)
+				as rel
+			from 
+				document
+			where 
+				match (internal_id, title, subtitle, subsubtitle, overview) 
+				against (:search in boolean mode)
+			order by
+				rel desc
+			limit
+				10
+		", ["search" => $_POST['search']]);
+		while ($r = DB::getInstance()->nextRecord($q)) {
+			$o = new Document($r->id);
+			$out->push($o);
+		}
+		return $out;
+	}
+
+	static public function search($page = 0) {
+		if (array_key_exists("search", $_POST)) {
+			$q = DB::getInstance()->query("
+				select
+					revision.id as id,
+					match(ocr.body) against (:search) as relevance
+				from
+					revision,ocr
+				where
+					match(ocr.body) against (:search) and
+					ocr.revision = revision.id and
+					not revision.document is null
+				order by
+					relevance desc
+	
+				", ["search" => $_POST['search']]);
+	
+			$slog = [];
+	
+			while ($r = DB::getInstance()->nextRecord($q)) {
+				$slog[] = $r->id;
+			}
+
+			Session::set("search", json_encode($slog));
+		}
+
+		$rpp = 8;
+
+		$offset = $page * $rpp;
+
+		$out = [];
+
+		$slog = json_decode(Session::get("search"));
+
+		for ($i = 0; $i < $rpp; $i++) {
+			if ($offset + $i < count($slog)) {
+				$rev = new Revision($slog[$offset + $i]);
+				if ($rev) {
+					$out[] = $rev;
+				}
+			}
+		}
+
+		return blade("search", ["page" => $page, "count" => count($slog), "results" => $out, "pages" => ceil(count($slog) / $rpp)]);
+	}
+}
+

app/controllers/SitemapController.php

@@ -0,0 +1,9 @@
+<?php
+
+class SitemapController {
+	public static function generate() {
+		$docs = Document::find()->all();
+		$revs = Revision::find()->all();
+		return blade("sitemap", ["docs" => $docs, "revs" => $revs]);
+	}
+}

app/controllers/SpiderController.php

@@ -0,0 +1,83 @@
+<?php
+
+class SpiderController {
+
+	public static function spider_pdfs() {
+		return blade("spider_pdfs");
+	}
+
+	public static function spider_pages() {
+		return blade("spider_pages");
+	}
+	
+	public static function api_pdfs() {
+		$pdfs = Spider::find([["status", "=", "N"]])->orderBy("id")->limit(20)->all();
+
+		return $pdfs;
+	}
+
+	public static function api_pages() {
+		$pages = SpiderPage::find([["status", "=", "O"], ["title", "!=", ""]])->orderBy("title")->limit(20)->all();
+
+		return $pages;
+	}
+
+	public static function api_reject_pdf($id = null) {
+		if ($id == null) return [];
+
+		$i = explode(",", $id);
+		foreach ($i as $id) {
+			$pdf = new Spider($id);
+			if ($pdf) {
+				$pdf->status = "B";
+				$pdf->save();
+			}
+		}
+		return SpiderController::api_pdfs();
+	}
+
+	public static function api_accept_pdf($_request, $id = null) {
+		if ($id == null) return [];
+
+		$i = explode(",", $id);
+		foreach ($i as $id) {
+			$pdf = new Spider($id);
+			if ($pdf) {
+				$pdf->status = "D";
+				$pdf->save();
+
+				$job = new DownloadJob($pdf->url);
+				$job->queue(get_user()->id);
+			}
+		}
+		return SpiderController::api_pdfs();
+	}
+
+	public static function api_reject_page($_request, $id) {
+		$i = explode(",", $id);
+		foreach ($i as $id) {
+			$page = new SpiderPage($id);
+			if ($page) {
+				$page->status = "B";
+				$page->save();
+			}
+		}
+		return SpiderController::api_pages();
+	}
+
+	public static function api_accept_page($id) {
+
+		$i = explode(",", $id);
+		foreach ($i as $id) {
+			$page = new SpiderPage($id);
+			if ($page) {
+				$page->status = "N";
+				$page->save();
+			}
+		}
+		return SpiderController::api_pages();
+	}
+}
+
+	
+

app/controllers/SystemController.php

@@ -0,0 +1,140 @@
+<?php
+
+class SystemController {
+	public static function status() {
+		$status = [
+			"B" => "Blacklisted",
+			"N" => "Pending",
+			"F" => "Failed",
+			"D" => "Done",
+			"Y" => "Done",
+			"W" => "Postponed",
+			"P" => "Processing",
+			"X" => "Deleted",
+			"O" => "Off-site",
+			"Q" => "Postponed",
+			"R" => "Redirect",
+		];
+
+		$q = DB::getInstance()->query("select count(id) as c, status from pages group by status order by status");
+		$spider = [];
+		while ($r = DB::getInstance()->nextRecord($q)) {
+			$spider[@$status[$r->status]] = $r->c;
+		}
+
+		$q = DB::getInstance()->query("select count(id) as c, status from spider group by status order by status");
+		$pdf = [];
+		while ($r = DB::getInstance()->nextRecord($q)) {
+			$pdf[@$status[$r->status]] = $r->c;
+		}
+
+		$q = DB::getInstance()->query("select count(id) as c, ocr from revision group by ocr order by ocr");
+		$ocr = [];
+		while ($r = DB::getInstance()->nextRecord($q)) {
+			$ocr[@$status[$r->ocr]] = $r->c;
+		}
+
+		$q = DB::getInstance()->query("select count(id) as c, idx from revision group by idx order by idx");
+		$idx = [];
+		while ($r = DB::getInstance()->nextRecord($q)) {
+			$idx[@$status[$r->idx]] = $r->c;
+		}
+
+		return blade("status", ["spider" => $spider, "pdf" => $pdf, "ocr" => $ocr, "idx" => $idx]);
+	}
+
+
+	public static function api_get_idmatches() {
+		return IDMatch::find([["id", ">=", 0]])->orderBy("weight")->all();
+	}
+
+	public static function api_add_idmatch($_request) {
+		$i = new IDMatch;
+		$i->example = $_request->put('example');
+		$i->regex = $_request->put('regex');
+		$i->weight = $_request->put('weight');
+		$i->save();
+		return SystemController::api_get_idmatches();
+	}
+
+	public static function api_set_idmatch($id) {
+		$i = new IDMatch($id);
+		if ($i) {
+			$i->example = $_POST['example'];
+			$i->regex = $_POST['regex'];
+			$i->weight = $_POST['weight'];
+			$i->save();
+		}
+		return SystemController::api_get_idmatches();
+	}
+
+	public static function api_del_idmatch($id) {
+		$i = new IDMatch($id);
+		if ($i) {
+			$i->delete();
+		}
+		return SystemController::api_get_idmatches();
+	}
+
+
+
+	public static function systems() {
+		$u = get_user();
+		$sys = System::find([["owner", "=", $u->id]])->all();
+		return blade("systems", ["systems" => $sys]);
+	}
+
+	public static function system($id) {
+		$sys = new System($id);
+		$sys->load("documents");
+		return blade("system", ["sys" => $sys]);
+	}
+
+	public static function systems_add() {
+		return blade("systems_add");
+	}
+
+	public static function systems_do_add($_request) {
+		$sys = new System;
+		$sys->name = $_request->post("name");
+		$sys->model = $_request->post("model");
+		$sys->notes = $_request->post("notes");
+		$sys->owner = get_user()->id;
+		$sys->save();
+		return redirect("/system/" . $sys->id);
+	}
+
+	public static function system_add_doc($id, $doc) {
+		$sys = new System($id);
+		if ($sys->owner == get_user()->id) {
+			$exist = SystemDoc::find([
+				["system", "=", $sys->id],
+				["document", "=", $doc]
+			])->first();
+			if (!$exist) {
+				$sd = new SystemDoc;
+				$sd->system = $sys->id;
+				$sd->document = $doc;
+				$sd->save();
+			}
+			return redirect("/system/" . $id);
+		}
+		return [403, "Forbidden"];
+	}
+
+	public static function system_del_doc($id, $doc) {
+		$sys = new System($id);
+		if ($sys->owner == get_user()->id) {
+			$sd = SystemDoc::find([
+				["system", "=", $sys->id],
+				["document", "=", $doc]
+			])->first();
+			if ($sd) {
+				$sd->delete();
+			}
+			return redirect("/system/" . $id);
+		}
+		return [403, "Forbidden"];
+	}
+}
+

app/jobs/DownloadJob.php

@@ -0,0 +1,87 @@
+<?php
+
+class DownloadJob extends Job {
+
+	public static function jobs() { return 3; }
+
+	public $from = null;
+	public $to = null;
+
+	private $_pct = 0;
+
+	public function __construct($from) {
+		$this->from = $from;
+		parent::__construct("download");
+	}
+
+	public function generate_to() {
+		$filename = ROOT . "/download/file-" . time() . "-" . rand() . ".pdf";
+		return $filename;
+	}
+	
+	public function run() {
+
+
+		$this->to = $this->generate_to();
+		$ch = curl_init();
+		$this->status("Downloading: 0%");
+		$fd = fopen($this->to, "w");
+
+		curl_setopt($ch, CURLOPT_URL, $this->from);
+		curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36");
+		curl_setopt($ch, CURLOPT_HEADER, 0);
+		curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 15);
+		curl_setopt($ch, CURLOPT_PRIVATE, $this);
+		curl_setopt($ch, CURLOPT_TIMEOUT, 3600);
+        curl_setopt($ch, CURLOPT_FILETIME, true);
+		curl_setopt($ch, CURLOPT_PROGRESSFUNCTION, [$this, 'download_progress']);
+		curl_setopt($ch, CURLOPT_NOPROGRESS, false);
+		curl_setopt($ch, CURLOPT_FILE, $fd);
+		curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
+
+		print("Running...\n");
+		$r = curl_exec($ch);
+		fclose($fd);
+
+		$file = new File($this->to);
+		$sha256 = $file->hash();
+
+		$rev = Revision::find([["sha256", "=", $sha256]])->first();
+
+		if ($rev) {
+			$this->status("Duplicate file");
+			$this->fail();
+			print("Finished (duplicate)\n");
+			return;
+		}
+
+		$u = new Upload();
+		$u->filename = basename($this->from);
+		$u->sha256 = $sha256;
+		$u->source = $this->from;
+		$u->ocr = "N";
+		$u->ai = "N";
+		$u->owner = $this->getOwner();
+		$u->save();
+		$file->rename(ROOT . "/uploads/" . $u->id . ".pdf");
+		$j = new ImportGeminiJob($u->id);
+		$j->queue($u->owner);
+		$j = new ImportOCRJob($u->id);
+		$j->queue($u->owner);
+
+		$this->status("Finished");
+		print("Finished\n");
+		$this->finish();
+	}
+
+
+
+	function download_progress($ch, $download_size, $downloaded, $upload_size, $uploaded) {
+		if ($download_size == 0) return;
+		$pct = round($downloaded / $download_size * 100);
+		if ($pct != $this->_pct) {
+			$this->status("Downloading: " . $pct . "%");
+			$this->_pct = $pct;
+		}
+	}
+}

app/jobs/GeminiJob.php

@@ -0,0 +1,211 @@
+<?php
+
+class GeminiJob extends Job {
+
+	public static function jobs() { return 1; }
+
+	public $docid;
+	public $move=false;
+
+	public function __construct($docid, $source="unknown") {
+		parent::__construct($source);
+		$this->docid = $docid;
+	}
+
+	public function run() {
+
+		$pdf = false;
+
+       $subs = [
+            "Installation and Operating Information",
+            "Installation and Configuration",
+            "Installing and Getting Started",
+            "Installation/Operator's Manual",
+            "Installation/Operator's Guide",
+            "Programmer's Reference Guide",
+			"Field Maintenance Print Set",
+            "Illustrated Parts Breakdown",
+            "Installation/Owner's Guide",
+            "Installation Information",
+            "Installation/User Guide",
+            "User Documentation Kit",
+            "Programmer Information",
+            "Technical Description",
+            "Operator Information",
+            "Configuration Guide",
+            "Upgrade Information",
+            "Installation Manual",
+            "Service Information",
+            "Installation Guide",
+			"Programming Manual",
+            "Maintenance Manual",
+            "Maintenance Guide",
+            "Technical Summary",
+            "Operator's Guide",
+            "System Reference",
+            "User Information",
+            "Technical Manual",
+            "Language Manual",
+            "Service Manual",
+            "Service Guide",
+            "Read Me First",
+            "Owner's Guide",
+            "Release Notes",
+            "Options Guide",
+            "Users' Manual",
+            "User's Manual",
+            "HiTest Notes",
+            "User's Guide",
+            "Design Guide",
+            "User Manual",
+            "User Guide",
+
+        ];
+
+		$doc = new Document($this->docid);
+
+		$minsize = 999999999999999;
+
+		foreach ($doc->revisions as $r) {
+			$p = new PDF($r->path() . "/doc.pdf");
+			if ($p->size() < $minsize) {
+				$minsize = $p->size();
+				$rev = $r;
+				$pdf = $p;
+			}
+		}
+
+		if (!$pdf) {
+			$this->fail("No files to upload");
+			return;
+		}
+
+		$prods = [];
+		$db = DB::getInstance();
+		$q = $db->query("select id, full_path from product");
+		while ($r = $db->nextRecord($q)) {
+			$prods[] = $r->id . "," . $r->full_path;
+		}
+
+		if ($doc->is_incoming()) {
+			$this->move = true;
+		}
+
+		file_put_contents("/tmp/category-list.csv", implode("\n", $prods) . "\n");
+
+		$this->status("Processing document with Gemini");
+		try {
+			$gemini = new Gemini();
+//			$gemini->verbose = true;
+
+			$p = new GeminiParameter("title", "object", "The title of the document (note: none of these parts should include the order number)", true);
+			$p->add_child("main_title", "string", "The primary portion of the title, properly recapitalized to fit English grammar", true);
+			$p->add_child("subtitle", "string", "Any portion of the title that could be used as a subtitle, such as 'user guide' or 'maintenance guide' etc. Remove this from the main title. Should not be the order number.");
+			$p->add_child("subsubtitle", "string", "Any part of the title that could be extra information such as version number or language. Should not be the order number.");
+			$gemini->add_parameter($p);
+
+			$p = new GeminiParameter("category", "integer", "The category the document best fits in from the provided list of categories in the file category-list.csv", true);
+			$gemini->add_parameter($p);
+
+			$gemini->add_parameter(new GeminiParameter("order_number", "string", "The document order number, often in the format XX-YYYYY-ZZ where YYYYY is based on the product code and ZZ is the type of document. Never more than 15 characters long."));
+			$gemini->add_parameter(new GeminiParameter("overview", "string", "A brief one-line overview of the document content in plain text format", true));
+			$gemini->add_parameter(new GeminiParameter("summary", "string", "A longer summary of the document content in markdown format", true));
+			$gemini->upload_callback([$this, "uploadcb"]);
+			$gemini->process_callback([$this, "processcb"]);
+
+			$gemini->attach($pdf->basename(), $pdf);
+			$gemini->attach("category-list.csv", new File("/tmp/category-list.csv"));
+
+			$lines = $gemini->geminiOverview();
+		} catch (Exception $e) {
+			$this->status($e->getMessage());
+			if ($e->getMessage() == "The request timed out. Please try again.") {
+				$this->retry("timeout");
+			} else if ($e->getMessage() == "HTTP Error 0 requesting AI assistance") {
+				$this->retry("error 0");
+			} else if (str_starts_with($e->getMessage(), "You exceeded your current quota, please check your plan and billing details.")) {
+				$this->retries++;
+				if ($this->retries < 5) {
+					$this->retry("quota exceeded");
+				} else {
+					$this->fail("Too many retries");
+				}
+			} else if ($e->getMessage() == "This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.") {
+				$this->retry("overload");
+			} else {
+				$this->fail();
+			}
+			return;
+		}
+
+		$data = json_decode($lines, true);
+
+		if (!$data) {
+			$this->fail("No response from AI");
+			return;
+		}
+
+		// Reload document since this took a while
+		$doc = new Document($this->docid);
+
+		if (array_key_exists("title", $data)) { 
+			$doc->subtitle = "";
+			$doc->subsubtitle = "";
+			if (array_key_exists("main_title", $data["title"])) { $doc->title = $data['title']['main_title']; }
+			if (array_key_exists("subtitle", $data["title"])) { $doc->subtitle = $data['title']['subtitle']; }
+			if (array_key_exists("subsubtitle", $data["title"])) { $doc->subsubtitle = $data['title']['subsubtitle']; }
+			if ($doc->subtitle != "") {
+				$doc->title = str_replace(" " . $doc->subtitle, "", $doc->title);
+			}
+		}
+		if (array_key_exists("order_number", $data)) { $doc->internal_id = substr($data['order_number'], 0, 20); }
+		if (array_key_exists("summary", $data)) { $doc->overview = $data['summary']; }
+		if (array_key_exists("overview", $data)) { $doc->oneliner = $data['overview']; }
+
+			
+
+		$doc->save();
+
+		if (array_key_exists("category", $data)) {
+			$cat = $data['category'];
+			$q = $db->query("select * from product where id=:id", ["id" => $cat]);
+			if ($r = $db->nextRecord($q)) {
+				if ($this->move) {
+					$q = $db->query("delete from docproduct where document=:id", ["id" => $doc->id]);
+				} 
+				$edp = DocProduct::find([
+					["product", "=", $r->id],
+					["document", "=", $doc->id]
+				])->first();
+
+				if (!$edp) {
+					$dp = new DocProduct;
+					$dp->document = $doc->id;
+					$dp->product = $r->id;
+					$dp->save();
+				}
+			} else {
+				print("--- Incorrect category suggested: " . $cat . "\n");
+			}
+		}
+
+		$this->status("Finished");
+		$this->finish();
+		return;
+	}
+
+
+	public function uploadcb($percent) {
+		$this->status("Uploading: " . $percent . "% complete");
+	}
+
+	public function processcb($message) {
+		$this->status($message);
+	}
+
+	function cleanup($txt) {
+		$txt = str_replace("Ø", "0", $txt);
+		$txt = str_replace(".", " ", $txt);
+		return $txt;
+	}
+}

app/jobs/ImportGeminiJob.php

@@ -0,0 +1,182 @@
+<?php
+
+class ImportGeminiJob extends Job {
+
+	public static function jobs() { return 1; }
+
+	public $upload_id;
+
+	public function __construct($upload_id) {
+		parent::__construct("upload:" . $upload_id);
+		$this->upload_id = $upload_id;
+	}
+
+	public function run() {
+
+		Config::refresh();
+
+		$pdf = false;
+
+       $subs = [
+            "Installation and Operating Information",
+            "Installation and Configuration",
+            "Installing and Getting Started",
+            "Installation/Operator's Manual",
+            "Installation/Operator's Guide",
+            "Programmer's Reference Guide",
+			"Field Maintenance Print Set",
+            "Illustrated Parts Breakdown",
+            "Installation/Owner's Guide",
+            "Installation Information",
+            "Installation/User Guide",
+            "User Documentation Kit",
+            "Programmer Information",
+            "Technical Description",
+            "Operator Information",
+            "Configuration Guide",
+            "Upgrade Information",
+            "Installation Manual",
+            "Service Information",
+            "Installation Guide",
+			"Programming Manual",
+            "Maintenance Manual",
+            "Maintenance Guide",
+            "Technical Summary",
+            "Operator's Guide",
+            "System Reference",
+            "User Information",
+            "Technical Manual",
+            "Language Manual",
+            "Service Manual",
+            "Service Guide",
+            "Read Me First",
+            "Owner's Guide",
+            "Release Notes",
+            "Options Guide",
+            "Users' Manual",
+            "User's Manual",
+            "HiTest Notes",
+            "User's Guide",
+            "Design Guide",
+            "User Manual",
+            "User Guide",
+
+        ];
+
+		$doc = new Upload($this->upload_id);
+		$doc->ai = "P";
+		$doc->save();
+		$pdf = $doc->getPDF();
+
+		if (!$pdf) {
+			$this->fail();
+			$doc->ai = "F";
+			$this->status("PDF File Missing");
+			$doc->save();
+			return;
+		}
+
+		if ($pdf->size() > 50000000) {
+			$this->status("File too large");
+			$doc->ai = "F";
+			$doc->save();
+			$this->fail();
+			return;
+		}
+
+		$this->status("Processing " . $doc->filename . " with Gemini");
+		try {
+			$gemini = new Gemini();
+			$gemini->upload_callback([$this, "uploadcb"]);
+			$gemini->process_callback([$this, "processcb"]);
+			$lines = $gemini->geminiOverview($pdf);
+		} catch (Exception $e) {
+			$mess = $e->getMessage();
+			print($mess . "\n");
+
+			switch ($mess) {
+				case "This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.":
+				case "The request timed out. Please try again.":
+				case "HTTP Error 0 requesting AI assistance":
+					$this->retry();
+					print("Retrying\n");
+					break;
+				default:
+					$this->fail();
+					$this->status(substr($mess, 0, 250));
+					$doc->overview = $mess;
+					$doc->ai = "F";
+					$doc->save();
+			}
+			return;
+		}
+
+		$this->status("Postprocessing returned data");
+		$ld = explode("\n", $lines);
+		if (preg_match('/^\{(.*)\}$/', trim($ld[0]), $m)) {
+			$title = trim($m[1]);
+
+			$title = str_replace("’", "'", $title);
+
+			$newsub = "";
+			$newsubsub = "";
+
+
+			$b = explode(":", $title);
+			if (count($b) > 1) {
+				$title = trim(array_shift($b));
+				$newsub = trim(array_shift($b));
+				$newsubsub = implode(": ", $b);
+			}
+
+			if ($newsub == "") {
+				foreach ($subs as $sub) {
+					if (str_ends_with($title, $sub)) {
+						$newsub = $sub;
+						$title = substr($title, 0, 0 - (strlen($sub) + 1));
+						break;
+					}
+				}
+			}
+		   $doc->title = $title;
+			$doc->subtitle = $newsub;
+			$doc->subsubtitle = $newsubsub;
+
+			array_shift($ld);
+			if (trim($ld[0]) == "") {
+				array_shift($ld);
+			}
+		}
+		if (preg_match('/^\[(.*)\]$/', trim($ld[0]), $m)) {
+
+			$iid = IDMatch::find_docid($this->cleanup($m[1]));
+
+			if ($iid) {
+				$doc->ai_docid = $iid[0];
+			}
+			array_shift($ld);
+		}
+		$doc->overview = implode("\n", $ld);
+		$doc->ai = "Y";
+
+		$doc->save();
+		$this->status("Finished");
+		$this->finish();
+		return;
+	}
+
+
+	public function uploadcb($percent) {
+		$this->status("Uploading: " . $percent . "% complete");
+	}
+
+	public function processcb($message) {
+		$this->status($message);
+	}
+
+	function cleanup($txt) {
+		$txt = str_replace("Ø", "0", $txt);
+		$txt = str_replace(".", " ", $txt);
+		return $txt;
+	}
+}

app/jobs/ImportOCRJob.php

@@ -0,0 +1,67 @@
+<?php
+
+class ImportOCRJob extends Job {
+
+	public static function jobs() { return 1; }
+
+	public $upload_id;
+
+	public function __construct($upload_id) {
+		parent::__construct("upload:" . $upload_id);
+		$this->upload_id = $upload_id;
+	}
+
+	public function run() {
+
+		$pdf = false;
+
+		$doc = new Upload($this->upload_id);
+		$doc->ocr = "P";
+		$doc->save();
+		$pdf = $doc->getPDF();
+
+		if (!$pdf) {
+			$this->fail();
+			$doc->ocr = "F";
+			$this->status("PDF File Missing");
+			$doc->save();
+			return;
+		}
+
+		$this->status("Processing " . $doc->filename . " with OCR");
+
+		$body = $pdf->text()->get_text();
+		if ($body == "") {
+			$this->status("Running OCR on PDF file");
+			$ocr = new PDF($pdf->path() . "-ocr.pdf");
+			$pdf->ocr($ocr);
+			$ocr->rename($pdf->path(), true);
+			$pdf = $doc->getPDF();
+			$body = $pdf->text()->get_text();
+		}
+
+		$doc->body = $body;
+
+		$docid = IDMatch::find_docid($doc->filename . " " . $body);
+		if ($docid) {
+			$doc->ocr_docid = $docid[0];
+		}
+
+		$doc->ocr = "Y";
+		$doc->save();
+
+		$this->status("Finished");
+		$this->finish();
+		return;
+	}
+
+
+	public function uploadcb($percent) {
+		$this->status("Uploading: " . $percent . "% complete");
+	}
+
+	public function processcb($message) {
+		$this->status($message);
+	}
+
+}

app/jobs/IndexJob.php

@@ -0,0 +1,47 @@
+<?php
+
+class IndexJob extends Job {
+
+	public static function jobs() { return 5; }
+
+	public $revision;
+
+
+	public function __construct($rev) {
+        parent::__construct("revision:" . $rev);
+		$this->revision = $rev;
+	}
+
+	public function run() {
+
+		$db = DB::getInstance();
+
+		$rev = new Revision($this->revision);
+		$rev->load("body");
+		$body = $rev->body;
+
+		if ($body) {
+			$body->counted = 'P';
+			$body->save();
+
+
+			$db->query("DELETE FROM word_index WHERE revision=:rev", ["rev" => $rev->id]);
+
+			$words = $body->count_words();
+
+			foreach ($words as $word=>$hits) {
+				$db->query("INSERT INTO word_index (revision, word, hits) VALUES (:rev, :word, :hits)", [
+					"rev" => $rev->id, 
+					"word" => substr($word, 0, 20),
+					"hits" => $hits
+				]);
+			}
+			$body->counted = 'Y';
+			$body->save();
+			$this->finish();
+		} else {
+			$this->fail();
+			$this->status("No body to count");
+		}
+	}
+}

app/jobs/OCRJob.php

@@ -0,0 +1,43 @@
+<?php
+
+class OCRJob extends Job {
+
+	public static function jobs() { return 1; }
+	public $revision;
+
+	public function __construct($rev) {
+		parent::__construct("revision:" . $rev);
+		$this->revision = $rev;
+	}
+
+	public function run() {
+		$ocr = OCR::find([["revision", "=", $this->revision]])->first();
+		if (!$ocr) {
+			$ocr = new OCR;
+			$ocr->revision = $this->revision;
+			$ocr->save();
+		}
+
+		$this->status("Processing");
+
+		$rev = new Revision($this->revision);
+		$pdf = $rev->getPDF();
+		$text = $pdf->text()->get_text();
+
+		print("[" . $text . "]\n");
+
+		if ($text == "") {
+			$this->status("Running OCR on PDF");
+			$newpdf = new PDF($pdf->parent() . "/ocr.pdf");
+			$pdf->ocr($newpdf);
+			$text = $newpdf->text()->get_text();
+		}
+
+		$ocr->body = $text;
+		$ocr->save();
+		$j = new IndexJob($this->revision);
+		$j->queue();
+		$this->status("Complete");
+		$this->finish();
+	}
+}

app/jobs/ProcessJob.php

@@ -0,0 +1,18 @@
+<?php
+
+class ProcessJob extends Job {
+
+	public static function jobs() { return 1; }
+
+	public $docid = 0;
+
+	public function __construct($docid) {
+		parent::__construct("document:$docid");
+		$this->docid = $docid;
+	}
+
+	public function run() {
+		return false;
+	}
+
+}

app/jobs/RecompressJob.php

@@ -0,0 +1,35 @@
+<?php
+
+class RecompressJob extends Job {
+
+	public static function jobs() { return 1; }
+
+    public $RevisionID;
+
+    public function __construct($id) {
+        parent::__construct("revision:" . $id);
+        $this->RevisionID = $id;
+    }
+	public function run() {
+		$this->status("Recompressing");
+		$rev = new Revision($this->RevisionID);
+		$crev = new Revision;
+
+		$crev->revid = $rev->revid;
+		$crev->save();
+
+		$from = $rev->getPDF();
+		$to = $crev->getPDF();
+
+		$to->parent()->mkdir();
+		$from->recompress($to->path());
+
+		$crev->document = $rev->document;
+		$crev->info = $to->info();
+		$crev->sha256 = $to->hash();
+		$crev->save();
+
+		$this->status("New revision " . $crev->id);
+		$this->finish();
+	}
+}

app/jobs/RedownloadJob.php

@@ -0,0 +1,59 @@
+<?php
+
+class RedownloadJob extends Job {
+
+	public static function jobs() { return 1; }
+
+	public $from = null;
+	public $to = null;
+	public $revid = null;
+
+	private $_pct = 0;
+
+	public function __construct($revid, $from, $to) {
+		$this->from = $from;
+		$this->to = $to;
+		$this->revid = $revid;
+		parent::__construct("revision:" . $revid);
+	}
+	
+	public function run() {
+		$ch = curl_init();
+		$this->status("Downloading: 0%");
+		$fd = fopen($this->to, "w");
+
+		curl_setopt($ch, CURLOPT_URL, $this->from);
+		curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36");
+		curl_setopt($ch, CURLOPT_HEADER, 0);
+		curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 15);
+		curl_setopt($ch, CURLOPT_PRIVATE, $this);
+		curl_setopt($ch, CURLOPT_TIMEOUT, 3600);
+        curl_setopt($ch, CURLOPT_FILETIME, true);
+		curl_setopt($ch, CURLOPT_PROGRESSFUNCTION, [$this, 'download_progress']);
+		curl_setopt($ch, CURLOPT_NOPROGRESS, false);
+		curl_setopt($ch, CURLOPT_FILE, $fd);
+		curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
+
+		print("Running...\n");
+		$r = curl_exec($ch);
+		fclose($fd);
+		$sha = hash_file("sha256", $this->to);
+		$rev = new Revision($this->revid);
+		$rev->sha256 = $sha;
+		$rev->save();
+		print("Finished\n");
+		$this->finish();
+		return;
+	}
+
+
+
+	function download_progress($ch, $download_size, $downloaded, $upload_size, $uploaded) {
+		if ($download_size == 0) return;
+		$pct = round($downloaded / $download_size * 100);
+		if ($pct != $this->_pct) {
+			$this->status("Downloading: " . $pct . "%");
+			$this->_pct = $pct;
+		}
+	}
+}

app/models/DocMeta.php

@@ -0,0 +1,13 @@
+<?php
+
+class DocMeta extends Model {
+
+    protected $_classes = [
+        "document" => "Document",
+        "metadata" => "MetaType"
+    ];
+
+
+	
+
+}

app/models/DocProduct.php

@@ -0,0 +1,17 @@
+<?php
+
+class DocProduct extends Model {
+	protected $_classes = [
+		"document" => "Document",
+		"product" => "Product"
+	];
+
+
+	protected $_model = [
+		"id" => [MODEL_SERIAL],
+		"created" => [MODEL_BIGINT, 11],
+		"updated" => [MODEL_BIGINT, 11],
+		"document" => [MODEL_OBJECT, "Document"],
+		"product" => [MODEL_OBJECT, "Product"]
+	];
+}

app/models/Document.php

@@ -0,0 +1,209 @@
+<?php
+
+class Document extends Model {
+
+	protected $_computed = [
+		"related" => "get_related",
+		"revisions" => "get_revisions",
+		"products" => "get_products",
+		"metadata" => "get_metadata",
+		"attachments" => "get_attachments",
+	];
+
+	public function get_revisions() {
+		return Revision::find([["document", "=", $this->id]])->orderByDesc("rating")->all();
+	}
+
+	public function get_related() {
+		$revs = Revision::find([["document", "=", $this->id]])->all();
+		$related = new Collection;
+		foreach ($revs as $rev) {
+			$q = DB::getInstance()->query("
+				select distinct 
+					id,
+					internal_id,
+					title,
+					subtitle,
+					subsubtitle 
+				from 
+					document 
+				where 
+					internal_id in (
+						select 
+							distinct words.word 
+						from 
+							words,
+							revwords 
+						where 
+							words.id=revwords.word 
+						and 
+							revwords.revision=:rev 
+						and 
+							words.word like '%-%-%'
+			)", array("rev" => $rev->id));
+			while ($r = DB::getInstance()->nextRecord($q)) {
+				if ($r->id != $this->id) {
+					$related->push(new Document($r->id));
+				}
+			}
+		}
+		return $related;
+	}
+
+    public function overview_md() {
+        return \Michelf\Markdown::defaultTransform($this->overview);
+    }
+
+	public function on_delete() {
+		DB::getInstance()->query("delete from docproduct where document=:id", ["id" => $this->id]);
+	}
+
+	private $_products = null;
+
+	public function get_products() {
+		if ($this->_products == null) {
+
+			$this->_products = new Collection;
+			$dpl = DocProduct::find([["document", "=", $this->id]])->all();
+			foreach ($dpl as $dp) {
+				$dp->load("product");
+				$this->_products->push($dp->product);
+			}
+		}
+
+		return $this->_products;
+	}
+
+	public function duplicate() {
+		$newdoc = new Document();
+		$newdoc->title = $this->title;
+		$newdoc->subtitle = $this->subtitle;
+		$newdoc->subsubtitle = $this->subsubtitle;
+		$newdoc->overview = $this->overview;
+		$newdoc->internal_id = $this->internal_id;
+		$newdoc->owner = $this->owner;
+		$newdoc->year = $this->year;
+		$newdoc->month = $this->month;
+
+		$newdoc->save();
+
+		$dpl = DocProduct::find([["document", "=", $this->id]])->all();
+		foreach ($dpl as $dp) {
+			$ndp = new DocProduct();
+			$ndp->document = $newdoc->id;
+			$ndp->product = $dp->product;
+			$ndp->save();
+		}
+		return $newdoc;
+	}
+
+	public function remove_product($id) {
+		$dp = DocProduct::find([["document", "=", $this->id], ["product", "=", $id]])->first();
+		if ($dp) {
+			$dp->delete();
+		}
+	}
+
+	public function set_metadata($metadata, $value) {
+		if (!$metadata) return false;
+		$m = DocMeta::find([["document", "=", $this->id], ["metadata", "=", $metadata]])->first();
+		if (!$m) {
+			$m = new DocMeta();
+			$m->document = $this->id;
+			$m->metadata = $metadata;
+		}
+		$m->data = $value;
+		$m->save();
+		return $m->id;
+	}
+
+	public function get_metadata() {
+        return DocMeta::find([["document", "=", $this->id]])->all();
+	}
+
+	public function remove_metadata($metadata) {
+		$m = DocMeta::find([["document", "=", $this->id], ["metadata", "=", $metadata]])->first();
+		if ($m) {
+			$m->delete();
+		}
+	}
+
+	public function get_metadata_by_id($metadata) {
+		$m = DocMeta::find([["document", "=", $this->id], ["metadata", "=", $metadata]])->first();
+		if (!$m) {
+			return "";
+		}
+		return $m->data;
+	}
+
+	public function guess_docid() {
+		$text = $this->title . " " . $this->subtitle . " " . $this->subsubtitle;
+		$text .= $this->overview;
+
+		$words = $this->get_words($text);
+
+		return IDMatch::find_docid($words)[0];
+	}
+
+
+	function get_words($text) {
+	    $text = str_replace("\r", " ", $text);
+	    $text = str_replace("\n\n", "\n", $text);
+	    $text = str_replace("\n\n", "\n", $text);
+	    $text = str_replace("\n\n", "\n", $text);
+	    $text = str_replace("\n\n", "\n", $text);
+	    $text = str_replace("\n", " ", $text);
+	    $text = str_replace("\"", "", $text);
+	    $text = str_replace(",", " ", $text);
+	    $text = str_replace("—", "-", $text);
+	    $text = str_replace("~", "-", $text);
+	    $text = str_replace("--", "-", $text);
+	
+	    $text = preg_replace('/([^a-zA-Z\-0-9]+)/', ' ', $text);
+	
+	    //$text = preg_replace("/([A-Za-z])(- )/", '$1', $text);
+	
+	    $words = preg_split('/\s+/', $text);
+	    return $words;
+	}
+
+
+    function get_attachments() {
+        $ap = ROOT . "/attachments/" . $this->id;
+        $files = new Collection;
+        if (!file_exists($ap)) {
+            return $files;
+        }
+        if (!is_dir($ap)) {
+            return $files;
+        }
+
+        $dir = opendir($ap);
+        while ($f = readdir($dir)) {
+            if (substr($f, 0, 1) == ".") continue;
+            $f = new File($ap . "/" . $f);
+            $files->push($f);
+        }
+    
+        return $files;
+    }
+
+	public static function find_by_docid($docid) {
+		$db = DB::getInstance();
+		$q = $db->query("select id from document where internal_id=:docid", ["docid" => $docid]);
+		if ($r = $db->nextRecord($q)) {
+			return new Document($r->id);
+		}
+		return false;
+	}
+
+	public function is_incoming() {
+		foreach ($this->get_products() as $prod) {
+			if (preg_match("/ Incoming /", $prod->full_path, $m)) {
+				return true;
+			}
+		}
+		return false;
+	}
+
+}

app/models/Favourite.php

@@ -0,0 +1,4 @@
+<?php
+
+class Favourite extends Model {
+}

app/models/IDMatch.php

@@ -0,0 +1,51 @@
+<?php
+
+class IDMatch extends Model {
+
+	public static function find_docid($words) {
+		if (!is_array($words)) {
+			$words = IDMatch::get_words($words);
+		}
+		$matches = IDMatch::find()->orderBy("weight")->all();
+
+		foreach ($matches as $match) {
+			$preg = '/^' . $match->regex . '$/';
+
+			foreach ($words as $word) {
+				$word = strtoupper($word);
+				$word = str_replace("(", "C", $word);
+				$word = str_replace("=", "-", $word);
+				$word = str_replace("--", "-", $word);
+
+				if (preg_match($preg, $word, $m)) {
+					return [$m[1], @$m[2]];
+				}
+			}
+		}
+		return false;
+	}
+
+    public static function get_words($text) {
+        $text = str_replace("\r", " ", $text);
+        $text = str_replace("\n\n", "\n", $text);
+        $text = str_replace("\n\n", "\n", $text);
+        $text = str_replace("\n\n", "\n", $text);
+        $text = str_replace("\n\n", "\n", $text);
+        $text = str_replace("\n", " ", $text);
+        $text = str_replace("\"", "", $text);
+        $text = str_replace(",", " ", $text);
+        $text = str_replace("–", "-", $text);
+        $text = str_replace("—", "-", $text);
+        $text = str_replace("~", "-", $text);
+        $text = str_replace("--", "-", $text);
+
+        $text = preg_replace('/([^a-zA-Z\-0-9]+)/', ' ', $text);
+
+        //$text = preg_replace("/([A-Za-z])(- )/", '$1', $text);
+
+        $words = preg_split('/\s+/', $text);
+        return $words;
+    }
+
+
+}

app/models/MetaType.php

@@ -0,0 +1,11 @@
+<?php
+
+class MetaType extends Model {
+	protected $table = "metatypes";
+
+
+	public static function name($id) {
+		$m = new MetaType($id);
+		return $m->name;
+	}
+}

app/models/OCR.php

@@ -0,0 +1,23 @@
+<?php
+
+class OCR extends Model {
+
+	public function get_words() {
+		$t = new Text($this->body);
+		return $t->words();
+	}
+
+	public function count_words() {
+		$words = $this->get_words();
+
+		$counted = [];
+
+		foreach ($words as $word) {
+			if (strlen($word) > 2) {
+				@$counted[strtoupper($word)] ++;
+			}
+		}
+
+		return $counted;
+	}
+}

app/models/Product.php

@@ -0,0 +1,163 @@
+<?php
+
+class Product extends Model {
+
+	protected $_classes = [
+		"parent" => "Product"
+	];
+
+	protected $_computed = [
+		"documents" => "get_documents",
+		"children" => "get_children",
+	];
+
+	protected $_triggers = [
+		"parent" => "update_path",
+		"title" => "update_path",
+	];
+
+	private $_children = null;
+	public function get_children() {
+		$c = $this->cache_get("children");
+		if ($c) return $c;
+
+		if ($this->_children == null) {
+			$this->_children = Product::find([["parent", "=", $this->id]])->orderBy("title")->all();
+		}
+		$this->cache_set("children", $this->_children);
+		return $this->_children;
+	}
+
+	public function get_full_title() {
+		$tree = $this->get_tree();
+
+		$n = [];
+		foreach ($tree as $t) {
+			$n[] = $t->title;
+		}
+
+		$out = implode(" / ", $n);
+		return $out;
+	}
+
+	public function update_path($ppath = null) {
+		if ($ppath == null) {
+			$this->full_path = $this->get_full_title();
+		} else {
+			$this->full_path = $ppath . " / " . $this->title;
+		}
+		$this->full_path = str_replace("/ / ", "/ ", $this->full_path);
+		$this->save();
+		foreach ($this->get_children() as $child) {
+			$child->update_path($this->fill_path);
+		}
+	}
+
+	public function get_tree() {
+		$out = [];
+		if ($this->load("parent")) {
+			$p = $this->parent;
+			$out = $p->get_tree();
+		}
+		array_push($out, $this);
+
+		return $out;
+	}
+
+	public function overview_md() {
+		return \Michelf\Markdown::defaultTransform($this->overview);
+	}
+
+    public function on_delete() {
+        DB::getInstance()->query("delete from docproduct where product=:id", ["id" => $this->id]);
+    }
+
+	private $_documents = null;
+
+	public function get_documents() {
+
+		$d = $this->cache_get("documents");
+		if ($d) return $d;
+
+		if ($this->_documents == null) {
+			//$dpl = DocProduct::find([["product", "=", $this->id]])->limit(100)->all();
+			$dpl = DocProduct::find([["product", "=", $this->id]])->all();
+			$this->_documents = new Collection;
+			foreach ($dpl as $dp) {
+				if ($dp->load("document")) {
+					$this->_documents->push($dp->document);
+				}
+			}
+
+			$this->_documents->sort("subsubtitle", true);
+			$this->_documents->sort("subtitle", true);
+			$this->_documents->sort("title", true);
+		}
+		$this->cache_set("documents", $this->_documents);
+		return $this->_documents;
+	}
+
+	public function add_document($doc) {
+		$dp = new DocProduct;
+		$dp->product = $this->id;
+		$dp->document = $doc->id;
+		$dp->save();
+	}
+
+	public function meta() {
+		if ($this->metadata == null) {
+			return [];
+		}
+		return explode(",", $this->metadata);
+	}
+
+	public function add_meta($id, $save = true) {
+		$m = $this->meta();
+		if (!in_array($id, $m)) {
+			$m[] = $id;
+		}
+		$this->metadata = implode(",", $m);
+		if ($save) $this->save();
+	}
+
+	public function del_meta($id, $save = true) {
+		$m = $this->meta();
+		$o = [];
+		foreach ($m as $v) {
+			if ($v != $id) {
+				$o[] = $v;
+			}
+		}
+		$this->metadaya = implode(",", $o);
+		if ($save) $this->save();
+	}
+
+	public function documents_sorted_by_meta() {
+
+		if ($this->metadata == null) {
+			$meta = [];
+		} else {
+			$meta = explode(",", $this->metadata);
+		}
+
+		$docs = $this->get_documents();
+
+
+		while (count($meta) > 0) {
+			$mid = array_pop($meta);
+			$docs->sort_with_function( function($a, $b) use ($mid) {
+				$va = $a->get_metadata_by_id($mid);
+				$vb = $b->get_metadata_by_id($mid);
+
+				if ($va > $vb) return 1;
+				if ($va < $vb) return -1;
+				return 0;
+				});
+		}
+
+
+		return $docs;
+
+	}
+
+}

app/models/Revision.php

@@ -0,0 +1,123 @@
+<?php
+
+class Revision extends Model {
+
+	protected $_classes = [
+		"document" => "Document"
+	];
+
+	protected $_computed = [
+		"body" => "get_body",
+	];
+
+	protected $transform = [
+		"info" => "json"
+	];
+
+	function path() {
+		$rs = sprintf("%011d", $this->id);
+		$d1 = substr($rs, 0, 2);
+		$d2 = substr($rs, 2, 3);
+		$d3 = substr($rs, 5, 3);
+		$d4 = substr($rs, 8, 3);
+		$p = sprintf("%s/pdf/%s/%s/%s/%s", ROOT, $d1, $d2, $d3, $d4);
+		return $p;
+	}
+
+	function filename() {
+		if ($this->document) {
+			if (is_numeric($this->document)) {
+				$this->load("document");
+			}
+			$out = $this->document->internal_id;
+			if (($this->revno != "") && ($this->revno != "0")) {
+				$out .= "-";
+				$out .= $this->revno;
+			}
+			$out .= " ";
+			$out .= $this->document->title . " " . $this->document->subtitle . " " . $this->document->subsubtitle;
+		} else {
+			$out = "doc";
+		}
+		$out = trim($out);
+		$out.= ".pdf";
+		$out = str_replace(" ", "_", $out);
+		$out = str_replace("/", "_", $out);
+		return $out;
+	}
+
+	function create_cover() {
+		$p = $this->path();
+		$pdf = new PDF($p . "/doc.pdf");
+		$cover = $p . "/cover.jpg";
+		$pdf->extract_page(0, $cover);
+	}
+
+	function cover($size = null) {
+		$f = new File($this->path() . "/cover.jpg");
+		if (!$f->exists()) {
+			$this->create_cover();
+		}
+		if ($size == null) {
+			return "/cover/" . $this->id . "/cover.jpg";
+		}
+		return "/cover/" . $this->id . "/" . $size . "/cover.jpg";
+	}
+
+	private $_body = null;
+
+	public function get_body() {
+		if ($this->_body == null) {
+			$ocr = OCR::find([["revision", "=", $this->id]])->first();
+			if (!$ocr) {
+				return false;
+			}
+			$this->_body = $ocr;
+		}
+		return $this->_body;
+	}
+
+	public function get_page($page, $dpi = 300) {
+		$file = new File(sprintf("%s/pages/%04d-%d.jpg", $this->path(), $page, $dpi));
+		if ($file->exists()) {
+			return new Image($file);
+		}
+
+		$dir = sprintf("%s/pages", $this->path());
+		if (!file_exists($dir)) {
+			mkdir($dir, 0777);
+		}
+
+		$pdf = new PDF(sprintf("%s/doc.pdf", $this->path()));
+		$img = $pdf->extract_page($page, $file->path(), $dpi);
+
+		if ($img) {
+			return $img;
+		}
+		return null;
+	}
+
+	public function purge() {
+		$dir = opendir($this->path());
+		while ($file = readdir($dir)) {
+			if (str_ends_with($file, ".jpg")) {
+				unlink($this->path() . "/" . $file);
+			}
+		}
+		closedir($dir);
+
+		$dir = opendir($this->path() . "/pages");
+		while ($file = readdir($dir)) {
+			if (str_ends_with($file, ".jpg")) {
+				unlink($this->path() . "/pages/" . $file);
+			}
+		}
+		closedir($dir);
+	}
+
+	public function getPDF() {
+        $p = $this->path();
+        $pdf = new PDF($p . "/doc.pdf");
+		return $pdf;
+	}
+}

app/models/Spider.php

@@ -0,0 +1,4 @@
+<?php
+
+class Spider extends Model {
+}

app/models/SpiderBanned.php

@@ -0,0 +1,4 @@
+<?php
+
+class SpiderBanned extends Model {
+}

app/models/SpiderDom.php

@@ -0,0 +1,4 @@
+<?php
+
+class SpiderDom extends Model {
+}

app/models/SpiderPage.php

@@ -0,0 +1,5 @@
+<?php
+
+class SpiderPage extends Model {
+	protected $table="pages";
+}

app/models/System.php

@@ -0,0 +1,23 @@
+<?php
+
+class System extends Model {
+	protected $table = "systems";
+
+    protected $_computed = [
+        "documents" => "get_documents",
+    ];
+
+	public function get_documents() {
+		$docs = SystemDoc::find([["system", "=", $this->id]])->all();
+		$dl = new Collection;
+		foreach ($docs as $d) {
+			$doc = new Document($d->document);
+			if ($doc) {
+				$dl->add($doc);
+			}
+		}
+		$dl->sort("title");
+		return $dl;
+	}
+
+}

app/models/SystemDoc.php

@@ -0,0 +1,5 @@
+<?php
+
+class SystemDoc extends Model {
+	protected $table = "systemdocs";
+}

app/models/Upload.php

@@ -0,0 +1,8 @@
+<?php
+
+class Upload extends Model {
+
+	public function getPDF() {
+		return new PDF(ROOT . "/uploads/" . $this->id . ".pdf");
+	}
+}

app/models/User.php

@@ -0,0 +1,43 @@
+<?php
+
+class User extends Model {
+	public static $table = "users";
+
+	public function is_favourite($doc) {
+		$f = Favourite::find([
+			["owner", "=", $this->id], 
+			["document", "=", $doc]])->first();
+		if ($f) {
+			return true;
+		}
+		return false;
+	}
+
+	public function set_favourite($doc) {
+		$f = Favourite::find([
+			["owner", "=", $this->id], 
+			["document", "=", $doc]])->first();
+		if (!$f) {
+			$f = new Favourite;
+			$f->owner = $this->id;
+			$f->document = $doc;
+			$f->save();
+		}
+	}
+
+	public function unset_favourite($doc) {
+		$f = Favourite::find([
+			["owner", "=", $this->id], 
+			["document", "=", $doc]])->first();
+		if ($f) {
+			$f->delete();
+		}
+	}
+
+	public function favourites() {
+		$f = Favourite::find([
+			["owner", "=", $this->id], 
+		])->all();
+		return $f;
+	}
+}