]> ruin.nu Git - ndwebbie.git/commitdiff
rework of scan storage and parsing
authorMichael Andreen <harv@ruin.nu>
Fri, 28 Dec 2007 08:11:55 +0000 (09:11 +0100)
committerMichael Andreen <harv@ruin.nu>
Fri, 28 Dec 2007 08:11:55 +0000 (09:11 +0100)
database/scans.sql [new file with mode: 0644]
scripts/scans.pl

diff --git a/database/scans.sql b/database/scans.sql
new file mode 100644 (file)
index 0000000..9682223
--- /dev/null
@@ -0,0 +1,97 @@
+ALTER TABLE fleets ADD COLUMN sender INTEGER NOT NULL REFERENCES planets(id) ON UPDATE CASCADE ON DELETE CASCADE;
+
+ALTER TABLE fleets DROP COLUMN fleet;
+
+ALTER TABLE fleets ALTER COLUMN target DROP NOT NULL;
+
+ALTER TABLE fleets ALTER COLUMN back DROP NOT NULL;
+
+ALTER TABLE fleets ALTER COLUMN eta DROP NOT NULL;
+
+ALTER TABLE fleets ADD COLUMN amount INTEGER;
+
+ALTER TABLE fleets ADD COLUMN name TEXT NOT NULL;
+
+ALTER TABLE fleets ADD COLUMN ingal BOOLEAN NOT NULL DEFAULT false;
+
+ALTER TABLE fleets RENAME COLUMN landing_tick TO tick;
+
+ALTER TABLE fleet_ships RENAME COLUMN fleet TO id;
+
+
+CREATE TABLE fleet_scans (
+       id INTEGER PRIMARY KEY REFERENCES fleets(id),
+       scan INTEGER NOT NULL REFERENCES scans(id)
+) WITHOUT OIDS;
+
+ALTER TABLE scans DROP COLUMN scan;
+
+ALTER TABLE scans DROP COLUMN type;
+
+ALTER TABLE scans ADD COLUMN type TEXT;
+
+ALTER TABLE scans ADD COLUMN uid INTEGER NOT NULL DEFAULT -1 REFERENCES users(uid) ON UPDATE RESTRICT ON DELETE RESTRICT;
+
+ALTER TABLE scans ADD COLUMN groupscan BOOLEAN NOT NULL DEFAULT False;
+
+ALTER TABLE scans ADD COLUMN parsed BOOLEAN NOT NULL DEFAULT False;
+
+ALTER TABLE scans ADD COLUMN id SERIAL PRIMARY KEY;
+
+ALTER TABLE scans ADD UNIQUE (scan_id, tick, groupscan);
+
+CREATE OR REPLACE FUNCTION planetid(x integer, y integer, z integer, tick integer) RETURNS integer
+    AS $_$SELECT id FROM planet_stats WHERE x = $1 AND y = $2 AND z = $3 AND (tick >= $4  OR tick =( SELECT max(tick) FROM planet_stats)) ORDER BY tick ASC LIMIT 1$_$
+    LANGUAGE sql STABLE;
+
+CREATE OR REPLACE FUNCTION planetcoords(IN id integer,IN tick integer, OUT x integer,OUT y integer,OUT z integer)
+    AS $_$SELECT x,y,z FROM planet_stats WHERE id = $1 AND (tick >= $2  OR tick =( SELECT max(tick) FROM planet_stats))  ORDER BY tick ASC LIMIT 1$_$
+    LANGUAGE sql STABLE;
+
+CREATE TABLE planet_data_types (
+       id SERIAL PRIMARY KEY,
+       category TEXT NOT NULL,
+       name TEXT NOT NULL,
+       UNIQUE (category,name)
+) WITHOUT OIDS;
+
+INSERT INTO planet_data_types (category,name) VALUES('roid','Metal');
+INSERT INTO planet_data_types (category,name) VALUES('roid','Crystal');
+INSERT INTO planet_data_types (category,name) VALUES('roid','Eonium');
+INSERT INTO planet_data_types (category,name) VALUES('resource','Metal');
+INSERT INTO planet_data_types (category,name) VALUES('resource','Crystal');
+INSERT INTO planet_data_types (category,name) VALUES('resource','Eonium');
+INSERT INTO planet_data_types (category,name) VALUES('tech','Space Travel');
+INSERT INTO planet_data_types (category,name) VALUES('tech','Infrastructure');
+INSERT INTO planet_data_types (category,name) VALUES('tech','Hulls');
+INSERT INTO planet_data_types (category,name) VALUES('tech','Waves');
+INSERT INTO planet_data_types (category,name) VALUES('tech','Core Extraction');
+INSERT INTO planet_data_types (category,name) VALUES('tech','Covert Ops');
+INSERT INTO planet_data_types (category,name) VALUES('tech','Asteroid Mining');
+INSERT INTO planet_data_types (category,name) VALUES('struc','Light Factory');
+INSERT INTO planet_data_types (category,name) VALUES('struc','Medium Factory');
+INSERT INTO planet_data_types (category,name) VALUES('struc','Heavy Factory');
+INSERT INTO planet_data_types (category,name) VALUES('struc','Wave Amplifier');
+INSERT INTO planet_data_types (category,name) VALUES('struc','Wave Distorter');
+INSERT INTO planet_data_types (category,name) VALUES('struc','Metal Refinery');
+INSERT INTO planet_data_types (category,name) VALUES('struc','Crystal Refinery');
+INSERT INTO planet_data_types (category,name) VALUES('struc','Eonium Refinery');
+INSERT INTO planet_data_types (category,name) VALUES('struc','Research Laboratory');
+INSERT INTO planet_data_types (category,name) VALUES('struc','Finance Centre');
+INSERT INTO planet_data_types (category,name) VALUES('struc','Security Centre');
+
+CREATE TABLE planet_data (
+       id SERIAL PRIMARY KEY,
+       uid INTEGER NOT NULL REFERENCES users(uid),
+       planet INTEGER NOT NULL REFERENCES planets(id),
+       tick INTEGER NOT NULL,
+       rid INTEGER NOT NULL REFERENCES planet_data_types(id),
+       amount INTEGER NOT NULL
+) WITHOUT OIDS;
+
+CREATE TABLE data_scans (
+       id INTEGER PRIMARY KEY REFERENCES planet_data(id),
+       scan INTEGER NOT NULL REFERENCES scans(id)
+) WITHOUT OIDS;
+
+DROP TABLE intel;
index 03d560d21f91bda24f4978d470dc30908496d0b2..46ffadecb351be63f805c3c5a3d581b33bb1e7f8 100755 (executable)
@@ -1,6 +1,6 @@
 #!/usr/bin/perl
 #**************************************************************************
-#   Copyright (C) 2006 by Michael Andreen <harvATruinDOTnu>               *
+#   Copyright (C) 2006,2007 by Michael Andreen <harvATruinDOTnu>          *
 #                                                                         *
 #   This program is free software; you can redistribute it and/or modify  *
 #   it under the terms of the GNU General Public License as published by  *
@@ -25,242 +25,187 @@ use CGI;
 use DBI;
 use DBD::Pg qw(:pg_types);
 use LWP::Simple;
+use lib qw{/var/www/ndawn/};
+use ND::DB;
 
-our $dbh;
-for my $file ("/home/whale/db.pl")
-{
-       unless (my $return = do $file){
-               warn "couldn't parse $file: $@" if $@;
-               warn "couldn't do $file: $!"    unless defined $return;
-               warn "couldn't run $file"       unless $return;
-       }
-}
+our $dbh = ND::DB::DB();
 
-$dbh->trace("3","/tmp/scanstest");
+#$dbh->trace("1","/tmp/scanstest");
+
+#my $test = $dbh->prepare(q{INSERT INTO scans (tick,scan_id) VALUES(1,3) RETURNING id});
+#print ;
 $dbh->do("SET CLIENT_ENCODING TO 'LATIN1';");
 
-my $scangroups = $dbh->prepare(q{SELECT scan_id,tick,scan FROM scans WHERE "type" = 'group' AND scan ~ '^-?[0-9]+$'});
+my $scangroups = $dbh->prepare(q{SELECT id,scan_id,tick,uid FROM scans WHERE groupscan AND NOT parsed});
 my $oldscan = $dbh->prepare(q{SELECT scan_id FROM scans WHERE scan_id = ? AND tick >= tick() - 168});
-my $addScan = $dbh->prepare(q{INSERT INTO scans (scan_id,tick,"type") VALUES (?,?,COALESCE(?,'-1'))});
-my $donegroup = $dbh->prepare(q{UPDATE scans SET "type" = 'donegroup' WHERE scan_id = ? AND "type" = 'group' AND tick = ?});
+my $addScan = $dbh->prepare(q{INSERT INTO scans (scan_id,tick,uid) VALUES (?,?,COALESCE(?,'-1'))});
+my $parsedscan = $dbh->prepare('UPDATE scans SET tick = ?, type = ?, planet = ?, parsed = TRUE WHERE id = ?');
+my $addpoints = $dbh->prepare('UPDATE users SET scan_points = scan_points + ? WHERE uid = ? ');
+my $delscan = $dbh->prepare('DELETE FROM scans WHERE id = ?');
 
-$scangroups->execute;
+$scangroups->execute or die $dbh->errstr;
 
-while (my @group = $scangroups->fetchrow){
-       my $file = get("http://game.planetarion.com/showscan.pl?scan_grp=$group[0]");
+while (my $group = $scangroups->fetchrow_hashref){
+       $dbh->being_work;
+       my $file = get("http://game.planetarion.com/showscan.pl?scan_grp=$group->{scan_id}");
 
        my $points = 0;
        while ($file =~ m/showscan.pl\?scan_id=(\d+)/g){
                unless ($dbh->selectrow_array($oldscan,undef,$1)){
-                       $addScan->execute($1,$group[1],$group[2]);
+                       $addScan->execute($1,$group->{tick},$group->{uid});
                        ++$points;
                }
        }
-       my $f = $dbh->prepare('UPDATE users SET scan_points = scan_points + ? WHERE uid = ? ');
-       $f->execute($points,$group[2]);
-       $donegroup->execute($group[0],$group[1]);
+       $addpoints->execute($points,$group->{uid});
+       $parsedscan->execute($group->{tick},'GROUP',$group->{id});
+       $dbh->commit;
 }
 
-my $emptyscans = $dbh->prepare('SELECT scan_id,tick,"type"::integer,tick() FROM scans WHERE planet is NULL AND type ~ \'^-?[0-9]+$\'');
-my $update = $dbh->prepare('UPDATE scans SET tick = ?, "type" = ?, scan = ? , planet = ? WHERE scan_id = ? AND tick = ?');
-$addScan = $dbh->prepare('INSERT INTO scans (tick,scan_id,"type",scan,planet) VALUES($1,$2,$3,$4,$5)') or die $dbh->errstr;
+my $newscans = $dbh->prepare('SELECT id,scan_id,tick,uid FROM scans WHERE NOT groupscan AND NOT parsed');
 my $findplanet = $dbh->prepare('SELECT planetid(?,?,?,?)');
+my $findcoords = $dbh->prepare('SELECT * FROM planetcoords(?,?)');
+my $addfleet = $dbh->prepare(q{INSERT INTO fleets (name,mission,sender,target,tick,eta,back,amount,ingal,uid) VALUES(?,?,?,?,?,?,?,?,?,-1) RETURNING id});
+my $fleetscan = $dbh->prepare(q{INSERT INTO fleet_scans (id,scan) VALUES(?,?)});
+my $addships = $dbh->prepare(q{INSERT INTO fleet_ships (id,ship,amount) VALUES(?,?,?)});
+my $addpdata = $dbh->prepare(q{INSERT INTO planet_data (planet,tick,rid,amount,uid) VALUES(?,?,(SELECT id FROM planet_data_types WHERE category = ? AND name = ?), ?,-1) RETURNING id});
+my $pdatascan = $dbh->prepare(q{INSERT INTO data_scans (id,scan) VALUES(?,?)});
 
-my $oldcoords = $dbh->prepare(q{SELECT x,y,z FROM planet_stats 
-       WHERE id = ? AND tick = ?});
-my $delscan = $dbh->prepare('DELETE FROM scans WHERE scan_id = ? AND tick = ?');
-unless ($emptyscans->execute){
-       my $cleanup = $dbh->prepare('UPDATE scans SET "type" = \'-1\' WHERE planet is NULL');
-       $cleanup->execute;
-       $emptyscans->execute;
-}
-while (my @scan = $emptyscans->fetchrow){
-       my $file = get("http://game.planetarion.com/showscan.pl?scan_id=$scan[0]");
+$newscans->execute or die $dbh->errstr;
+while (my $scan = $newscans->fetchrow_hashref){
+       my $file = get("http://game.planetarion.com/showscan.pl?scan_id=$scan->{scan_id}");
+       next unless defined $file;
        if ($file =~ /((?:\w| )*) (?:Scan|Probe) on (\d+):(\d+):(\d+) in tick (\d+)/){
+               $dbh->begin_work;
+               eval {
                my $type = $1;
                my $x = $2;
                my $y = $3;
                my $z = $4;
                my $tick = $5;
+
+               if($dbh->selectrow_array(q{SELECT * FROM scans WHERE scan_id = ? AND tick = ? AND id <> ?},undef,$scan->{scan_id},$tick,$scan->{id})){
+                       $dbh->rollback;
+                       $delscan->execute($scan->{id});
+                       $addpoints->execute(-1,$scan->{uid}) if $scan->{uid} > 0;
+                       die "Duplicate scan: $scan->{id} http://game.planetarion.com/showscan.pl?scan_id=$scan->{scan_id}\n";
+               }
                my ($planet) = $dbh->selectrow_array($findplanet,undef,$x,$y,$z,$tick);
                unless ($planet){
-                       if ($scan[1] + 48 < $scan[3]){
-                               $delscan->execute($scan[0],$scan[1]);
-                       }
+                       $dbh->rollback;
                        next;
                }
                my $scantext = "";
                if ($file =~ /(Note: [^<]*)/){
-                       $scantext .= qq{
-                       <table class="closedPlanet">
-                       <tr><td>$1</td></tr>
-                       </table>};
+                       #TODO: something about planet being closed?
                }
                if ($type eq 'Planet'){
                        $file =~ s/(\d),(\d)/$1$2/g;
-                       if($file =~ m/Metal\D+(\d+)\D+(\d+).+?Crystal\D+(\d+)\D+(\d+).+?Eonium\D+(\d+)\D+(\d+)/s){
-                               $scantext .= <<HTML
-       <table cellpadding="2">
-       <tr><th></th><th>Metal</th><th>Crystal</th><th>Eonium</th></tr>
-       <tr><td>Asteroids</td><td>$1</td><td>$3</td><td>$5</td></tr>
-       <tr><td>Resources</td><td>$2</td><td>$4</td><td>$6</td></tr>
-       </table>
-HTML
-                               }
-                       ;
-                       my $f = $dbh->prepare("UPDATE covop_targets SET metal = ?, crystal = ?, eonium = ? WHERE planet = ?");
-                       if ($f->execute($2,$4,$6,$planet) < 1){
-                               $f = $dbh->prepare("INSERT INTO covop_targets (planet,metal, crystal, eonium) VALUES(?,?,?,?)");
-                               $f->execute($planet,$2,$4,$6);
+                       while($file =~ m/(Metal|Crystal|Eonium)\D+(\d+)\D+(\d+)/g){
+                               my $id = $dbh->selectrow_array($addpdata,undef,$planet,$tick
+                                       ,'roid',$1, $2) or die $dbh->errstr;
+                               $pdatascan->execute($id,$scan->{id});
+                               $id = $dbh->selectrow_array($addpdata,undef,$planet,$tick
+                                       ,'resource',$1, $3) or die $dbh->errstr;
+                               $pdatascan->execute($id,$scan->{id});
                        }
                }elsif ($type eq 'Jumpgate'){
-                       $scantext .= <<HTML
-       <table>
-       <tr>
-       <th>Coords</th>
-       <th>Mission</th>
-       <th>Fleet</th>
-       <th>Eta</th>
-       <th>Amount</th>
-       </tr>
-HTML
-                       ;
-                       my $f = $dbh->prepare("SELECT add_intel(?,?,?,?,?,?,?,?,?,?,-1)");
-                       my $i = 1;
                        while ($file =~ m/(\d+):(\d+):(\d+)\D+"left"\>(Attack|Defend|Return)<\/td><td>([^<]*)<\/td><td>(\d+)\D+(\d+)/g){
-                               my $row = "odd";
-                               $row = "even" if ($i % 2 == 0);
-                               if ($4 ne 'Return'){
-                                       $f->execute($tick,$6,$x,$y,$z,$1,$2,$3,$7,$4);# or $server->command("echo " . DBI->errstr);
-                               }
-                               $scantext .= qq{
-       <tr class="$row"><td><a href="check?coords=$1:$2:$3">$1:$2:$3</a></td><td class="$4">$4</td><td>$5</td><td>$6</td><td>$7</td></tr>};
-                               $i++;
+                               
+                               my ($sender) = $dbh->selectrow_array($findplanet,undef,$1,$2,$3,$tick) or die $dbh->errstr;
+                               my $id = addfleet($5,$4,"",$sender,$planet,$tick+$6,$6
+                                       ,undef,$7, $x == $1 && $y == $2);
+                               $fleetscan->execute($id,$scan->{id}) or die $dbh->errstr;
                        }
-                       $scantext .= "</table>\n";
                }elsif ($type eq 'News'){
-                       $scantext .= "<table>\n";
-                       my $i = 1;
-                       my $cgi = new CGI;
-                       my $f = $dbh->prepare("SELECT add_intel(?,?,?,?,?,?,?,?,?,?,-1)");
                        while( $file =~ m{top">((?:\w| )+)\D+(\d+)</td><td class="left" valign="top">(.+?)</td></tr>}g){
-                               my $row = "odd";
-                               $row = "even" if ($i % 2 == 0);
-                               $i++;
                                my $news = $1;
                                my $t = $2;
-                               my $text = $cgi->escapeHTML($3);
-                               my $class = '';
-                               my ($x,$y,$z) = $dbh->selectrow_array($oldcoords,undef,$planet,$t);
-
-                               if($news eq 'Launch' && $text =~ m/(?:[^<]*) fleet has been launched, heading for (\d+):(\d+):(\d+), on a mission to (Attack|Defend). Arrival tick: (\d+)/g){
-
+                               my $text = $3;
+                               my ($x,$y,$z) = $dbh->selectrow_array($findcoords,undef,$planet,$t);
+                               die "No coords for: $planet tick $t" unless defined $x;
+                               if($news eq 'Launch' && $text =~ m/The (.*?) fleet has been launched, heading for (\d+):(\d+):(\d+), on a mission to (Attack|Defend). Arrival tick: (\d+)/g){
+                                       my $eta = $6 - $t;
+                                       my $mission = $5;
+                                       my $back = $6 + $eta;
+                                       $mission = 'AllyDef' if $eta == 6 && $x != $2;
+                                       my ($target) = $dbh->selectrow_array($findplanet,undef
+                                               ,$2,$3,$4,$t) or die $dbh->errstr;
+                                       die "No target: $2:$3:$4" unless defined $target;
+                                       my $id = addfleet($1,$mission,"",$planet,$target,$6
+                                               ,$eta,$back,undef, ($x == $2 && $y == $3));
+                                       $fleetscan->execute($id,$scan->{id}) or die $dbh->errstr;
+                               }elsif($news eq 'Incoming' && $text =~ m/We have detected an open jumpgate from (.*?), located at (\d+):(\d+):(\d+). The fleet will approach our system in tick (\d+) and appears to have roughly (\d+) ships/g){
                                        my $eta = $5 - $t;
-                                       my $mission = $4;
-                                       $mission = 'AllyDef' if $eta == 6 && $x != $1;
-                                       $f->execute($t,$eta,$1,$2,$3,$x,$y,$z,-1,$mission) or print $dbh->errstr;
-                                       $class = qq{ class="$mission"};
-                               }elsif($news eq 'Incoming' && $text =~ m/We have detected an open jumpgate from (?:[^<]*), located at (\d+):(\d+):(\d+). The fleet will approach our system in tick (\d+) and appears to have roughly (\d+) ships/g){
-                                       my $eta = $4 - $t;
                                        my $mission = '';
+                                       my $back = $5 + $eta;
                                        $mission = 'Defend' if $eta <= 6;
-                                       $mission = 'AllyDef' if $eta == 6 && $x != $1;
-                                       $f->execute($t,$eta,$x,$y,$z,$1,$2,$3,$5,$mission) or print $dbh->errstr;
-                                       $class = qq{ class="$mission"};
+                                       $mission = 'AllyDef' if $eta == 6 && $x != $2;
+                                       my ($target) = $dbh->selectrow_array($findplanet,undef
+                                               ,$2,$3,$4,$t) or die $dbh->errstr;
+                                       die "No target: $2:$3:$4" unless defined $target;
+                                       my $id = addfleet($1,$mission,"",$planet,$target,$5
+                                               ,$eta,$back,$6, ($x == $2 && $y == $3));
+                                       $fleetscan->execute($id,$scan->{id}) or die $dbh->errstr;
                                }
-                               $text =~ s{(\d+):(\d+):(\d+)}{<a href="check?coords=$1:$2:$3">$1:$2:$3</a>}g;
-                               $scantext .= "<tr class =\"$row\"><td$class>$news</td><td>$t</td><td>$text</td></tr>\n";
                        }
-                       $scantext .= "</table>\n";
-               } elsif($type eq 'Unit' || $type eq 'Advanced Unit' || $type eq 'Surface Analysis' || $type eq 'Technology Analysis'){
-                       $scantext .= "<table>\n";
-                       my $i = 0;
-                       my $total = 0;
-                       my $sec = 0;
-                       my $dist = 0;
-                       my $f = $dbh->prepare(qq{SELECT "type","class" FROM ship_stats WHERE name = ?});
-                       my %visible;
-                       my %total;
+               } elsif($type eq 'Surface Analysis' || $type eq 'Technology Analysis'){
+                       my $cat = ($type eq 'Surface Analysis' ? 'struc' : 'tech');
                        while($file =~ m{((?:[a-zA-Z]| )+)</t[dh]><td(?: class="right")?>(\d+)}sg){
-                               $i++;
-                               my $row = "odd";
-                               $row = "even" if ($i % 2 == 0);
-                               $scantext .= "<tr class=\"$row\"><td>$1</td><td>$2</td></tr>\n";
-                               $total += $2;
-                               $sec = $2 if ($1 eq 'Security Centre');
-                               $dist = $2 if ($1 eq 'Wave Distorter');
-                               $f->execute($1);
-                               if (my $ship = $f->fetchrow_hashref){
-                                       $total{$ship->{class}} += $2;
-                                       $visible{$ship->{class}} += $2 unless $ship->{type} eq 'Cloak';
-                               }
+                               my $id = $dbh->selectrow_array($addpdata,undef,$planet,$tick
+                                       ,$cat,$1, $2) or die $dbh->errstr;
+                               $pdatascan->execute($id,$scan->{id});
                        }
-                       if ($type =~ 'Unit'){
-                               my $scan .= q{<table>
-                                       <tr><th>Class</th><th>Total</th><th>Visible</th></tr>};
-                               my $i = 0;
-                               for my $type (qw/Fighter Corvette Frigate Destroyer Cruiser Battleship/){
-                                       next unless $total{$type};
-                                       $i++;
-                                       my $row = "odd";
-                                       $row = "even" if ($i % 2 == 0);
-                                       $visible{$type} = 0 unless $visible{$type};
-                                       $scan .= "<tr class=\"$row\"><td>$type</td><td>".$total{$type}."</td><td>".$visible{$type}."</td></tr>\n";
-                               }
-                               $scan .= "</table>\n";
-                               $addScan->execute($tick-1,$scan[0],'Ship Classes',$scan,$planet);
-                       }
-                       $scantext .= "<tr class=\"odd\"><td>No</td><td>Ships</td></tr>\n" unless $i;
-                       $scantext .= "</table>\n";
 
-                       if($type eq 'Surface Analysis'){
-                               my $f = $dbh->prepare("UPDATE covop_targets SET structures = ?, sec_centres = ?, dists = ? WHERE planet = ?");
-                               if ($f->execute($total,$sec,$dist,$planet) < 1){
-                                       $f = $dbh->prepare("INSERT INTO covop_targets (planet,structures, sec_centres, dists) VALUES(?,?,?,?)");
-                                       $f->execute($planet,$total,$sec,$dist);
-                               }
-                       }
-               } elsif($type eq 'Military'){
-                       $scantext .= "<table>\n";
-                       my $i = 1;
-                       my @totals = (0,0,0,0);
-                       my @eta = (8,8,8,8);
-                       my $f = $dbh->prepare(qq{SELECT "type","class" FROM ship_stats WHERE name = ?});
-                       while($file =~ m/big left">((?:[a-zA-Z]| )+)<\/t[dh]>.*?center>(\d+).*?center>(\d+).*?center>(\d+).*?center>(\d+)/sg){
-                               next if ($2+$3+$4+$5 == 0);
-                               my @ships = ($2,$3,$4,$5);
-                               my $row = "odd";
-                               my ($type,$class) = $dbh->selectrow_array($f,undef,$1);
-                               #print "$1 $type\n";
-                               $row = "even" if ($i % 2 == 0);
-                               $scantext .= "<tr class=\"$row\"><td>$1</td><td>$2</td><td>$3</td><td>$4</td><td>$5</td></tr>\n";
-                               $i++;
-                               unless ($type eq "Cloak"){
-                                       $totals[0] += $2;
-                                       $totals[1] += $3;
-                                       $totals[2] += $4;
-                                       $totals[3] += $5;
-                               }
-                               foreach my $i (0,1,2,3){
-                                       if ($ships[$i] > 0 && $eta[$i] < 9 && ($class =~ /Frigate|Destroyer/)){
-                                               $eta[$i] = 9;
-                                       }elsif ($ships[$i] > 0 && $eta[$i] < 10 && ($class =~ /Cruiser|Battleship/)){
-                                               $eta[$i] = 10;
-                                       }
-                               }
+               } elsif($type eq 'Unit' || $type eq 'Advanced Unit'){
+                       my $id = addfleet($type,'Full fleet',$file,$planet,undef,$tick,undef,undef,undef);
+                       $fleetscan->execute($id,$scan->{id}) or die $dbh->errstr;
+               } elsif($type eq 'Incoming'){
+                       while($file =~ m{class="left">Fleet: (.*?)</td><td class="left">Mission: (\w+)</td></tr>(.*?)Total Ships: (\d+)}sg){
+                               my $id = addfleet($1,$2,$3,$planet,undef,$tick,undef,undef,$4);
+                               $fleetscan->execute($id,$scan->{id}) or die $dbh->errstr;
                        }
-                       $scantext .= "<tr class=\"total\"><td>Total uncloaked</td><td>$totals[0]</td><td>$totals[1]</td><td>$totals[2]</td><td>$totals[3]</td></tr>\n";
-                       $scantext .= "<tr><td>Initial eta</td><td>$eta[0]</td><td>$eta[1]</td><td>$eta[2]</td><td>$eta[3]</td></tr>\n";
-                       $scantext .= "</table>\n";
+               } else {
+                       print "Something wrong with scan $scan->{id} type $type at tick $tick http://game.planetarion.com/showscan.pl?scan_id=$scan->{scan_id}";
                }
-               unless ($scantext || $type eq 'Incoming'){
-                       print "Something wrong with scan $scan[0] type $type at tick $tick";
+               $parsedscan->execute($tick,$type,$planet,$scan->{id}) or die $dbh->errstr;
+               $dbh->commit;
+               #$dbh->rollback;
+               };
+               if ($@) {
+                       warn $@;
+                       $dbh->rollback;
                }
-               $update->execute($tick,$type,$scantext,$planet,$scan[0],$scan[1]) or warn DBI->errstr;
        }else{
-               my $f = $dbh->prepare('UPDATE users SET scan_points = scan_points - 1 WHERE uid = ? ');
-               $f->execute($scan[2]);
-               $delscan->execute($scan[0],$scan[1]);
+               warn "Nothing useful in scan: $scan->{id} http://game.planetarion.com/showscan.pl?scan_id=$scan->{scan_id}\n";
+               $delscan->execute($scan->{id});
+               $addpoints->execute(-1,$scan->{uid}) if $scan->{uid} > 0;
        }
 }
-$dbh->disconnect;
+
+sub addfleet {
+       my ($name,$mission,$ships,$sender,$target,$tick,$eta,$back,$amount,$ingal) = @_;
+
+       $ingal = 0 unless $ingal;
+       $back = $tick + 4 if $ingal && $eta <= 4;
+
+       if ($mission eq 'Return'){
+               ($sender,$target) = ($target,$sender);
+               $back = $tick + $eta if $eta;
+       }
+
+       my @ships;
+       my $total = undef;
+       while($ships =~ m{((?:[a-zA-Z]| )+)</td><td>(\d+)}sg){
+               $total = 0 unless defined $total;
+               $total += $2;
+               push @ships, [$1,$2];
+       }
+       $amount = $total unless defined $amount;
+       my $id = $dbh->selectrow_array($addfleet,undef,$name,$mission,$sender
+               ,$target,$tick, $eta, $back, $amount,$ingal) or die $dbh->errstr;
+       for my $s (@ships){
+               unshift @{$s},$id;
+               $addships->execute(@{$s}) or die $dbh->errstr;
+       }
+       return $id;
+};