From 47f76da4f8cc31146d2834dfdf9731d330288c9d Mon Sep 17 00:00:00 2001 From: Birte Kristina Friesel Date: Fri, 26 Jul 2024 18:55:58 +0200 Subject: Multi-backend support Squashed commit of the following: commit 92518024ba295456358618c0e8180bd8e996fdf1 Author: Birte Kristina Friesel Date: Fri Jul 26 18:39:46 2024 +0200 add_or_update station: remove superfluos 'new backend id := old backend id' commit df21c20c6e4c86454f8a9ac69121404415217f2a Author: Birte Kristina Friesel Date: Fri Jul 26 18:35:51 2024 +0200 revert connection targets min_count to 3 commit be335cef07d0b42874f5fc1de4a1d13396e8e807 Author: Birte Kristina Friesel Date: Fri Jul 26 18:20:05 2024 +0200 mention backend selection in API documentation commit 9f41828fb4f18fd707e0087def3032e8d4c8d7d8 Author: Birte Kristina Friesel Date: Thu Jul 25 20:19:23 2024 +0200 use_history: not all backends provide route data in departure monitor commit 09714b4d89684b8331d0e96f564a4c7432318f70 Author: Birte Kristina Friesel Date: Thu Jul 25 20:11:44 2024 +0200 disambiguation: pass correct hafas parameter commit 8cdf1120fc32155dc6525be64601b7c10a9c7f52 Author: Birte Kristina Friesel Date: Thu Jul 25 20:11:28 2024 +0200 _checked_in: hide Zuglauf link for non-db checkins commit 7455653f541198e0e0a6d11aed421487ffdb6285 Author: Birte Kristina Friesel Date: Thu Jul 25 20:01:47 2024 +0200 debug output commit b9cda07f85601a58ea32dbdacdd5399f302db52b Author: Birte Kristina Friesel Date: Thu Jul 25 19:09:07 2024 +0200 fix remaining get_connection_targets / get_connecting_trains_p invocations commit 2759d7258c37c7498905cfe19f6b4c4f6d16bd21 Author: Birte Kristina Friesel Date: Wed Jul 24 20:50:12 2024 +0200 support non-DB HAFAS backends (WiP) --- lib/Travelynx/Model/InTransit.pm | 23 ++--- lib/Travelynx/Model/Journeys.pm | 67 +++++++----- lib/Travelynx/Model/Stations.pm | 203 +++++++++++++++++++++++++++++++------ lib/Travelynx/Model/Traewelling.pm | 1 + lib/Travelynx/Model/Users.pm | 15 ++- 5 files changed, 236 insertions(+), 73 deletions(-) (limited to 'lib/Travelynx/Model') diff --git a/lib/Travelynx/Model/InTransit.pm b/lib/Travelynx/Model/InTransit.pm index f5a1f6d..b9fbabc 100644 --- a/lib/Travelynx/Model/InTransit.pm +++ b/lib/Travelynx/Model/InTransit.pm @@ -93,6 +93,7 @@ sub add { my $uid = $opt{uid}; my $db = $opt{db} // $self->{pg}->db; + my $backend_id = $opt{backend_id}; my $train = $opt{train}; my $journey = $opt{journey}; my $stop = $opt{stop}; @@ -103,8 +104,6 @@ sub add { my $json = JSON->new; if ($train) { - my $backend_id - = $db->select( 'backends', ['id'], { iris => 1 } )->hash->{id}; $db->insert( 'in_transit', { @@ -136,14 +135,6 @@ sub add { ); } elsif ( $journey and $stop ) { - my $backend_id = $db->select( - 'backends', - ['id'], - { - hafas => 1, - name => 'DB' - } - )->hash->{id}; my @route; my $product = $journey->product_at( $stop->loc->eva ) // $journey->product; @@ -440,17 +431,20 @@ sub get_all_active { ->hashes->each; } -sub get_checkout_station_id { +sub get_checkout_ids { my ( $self, %opt ) = @_; my $uid = $opt{uid}; my $db = $opt{db} // $self->{pg}->db; - my $status = $db->select( 'in_transit', ['checkout_station_id'], - { user_id => $uid } )->hash; + my $status = $db->select( + 'in_transit', + [ 'checkout_station_id', 'backend_id' ], + { user_id => $uid } + )->hash; if ($status) { - return $status->{checkout_station_id}; + return $status->{checkout_station_id}, $status->{backend_id}; } return; } @@ -819,7 +813,6 @@ sub update_arrival_hafas { my $stop = $opt{stop}; my $json = JSON->new; - # TODO use old rt data if available my @route; for my $j_stop ( $journey->route ) { push( diff --git a/lib/Travelynx/Model/Journeys.pm b/lib/Travelynx/Model/Journeys.pm index b067d78..e47df58 100755 --- a/lib/Travelynx/Model/Journeys.pm +++ b/lib/Travelynx/Model/Journeys.pm @@ -545,7 +545,7 @@ sub get { my @select = ( - qw(journey_id is_iris is_hafas backend_name train_type train_line train_no checkin_ts sched_dep_ts real_dep_ts dep_eva dep_ds100 dep_name dep_lat dep_lon checkout_ts sched_arr_ts real_arr_ts arr_eva arr_ds100 arr_name arr_lat arr_lon cancelled edited route messages user_data visibility effective_visibility) + qw(journey_id is_iris is_hafas backend_name backend_id train_type train_line train_no checkin_ts sched_dep_ts real_dep_ts dep_eva dep_ds100 dep_name dep_lat dep_lon checkout_ts sched_arr_ts real_arr_ts arr_eva arr_ds100 arr_name arr_lat arr_lon cancelled edited route messages user_data visibility effective_visibility) ); my %where = ( user_id => $uid, @@ -606,6 +606,7 @@ sub get { is_iris => $entry->{is_iris}, is_hafas => $entry->{is_hafas}, backend_name => $entry->{backend_name}, + backend_id => $entry->{backend_id}, type => $entry->{train_type}, line => $entry->{train_line}, no => $entry->{train_no}, @@ -665,7 +666,10 @@ sub get { my $rename = $self->{renamed_station}; for my $stop ( @{ $ref->{route} } ) { if ( $stop->[0] =~ m{^Betriebsstelle nicht bekannt (\d+)$} ) { - if ( my $s = $self->{stations}->get_by_eva($1) ) { + if ( my $s + = $self->{stations} + ->get_by_eva( $1, backend_id => $ref->{backend_id} ) ) + { $stop->[0] = $s->{name}; } } @@ -800,14 +804,14 @@ sub get_oldest_ts { return undef; } -sub get_latest_checkout_station_id { +sub get_latest_checkout_ids { my ( $self, %opt ) = @_; my $uid = $opt{uid}; my $db = $opt{db} // $self->{pg}->db; my $res_h = $db->select( 'journeys', - ['checkout_station_id'], + [ 'checkout_station_id', 'backend_id', ], { user_id => $uid, cancelled => 0 @@ -822,7 +826,7 @@ sub get_latest_checkout_station_id { return; } - return $res_h->{checkout_station_id}; + return $res_h->{checkout_station_id}, $res_h->{backend_id}; } sub get_latest_checkout_stations { @@ -833,7 +837,10 @@ sub get_latest_checkout_stations { my $res = $db->select( 'journeys_str', - [ 'arr_name', 'arr_eva', 'train_id' ], + [ + 'arr_name', 'arr_eva', 'train_id', 'backend_id', + 'backend_name', 'is_hafas' + ], { user_id => $uid, cancelled => 0 @@ -854,9 +861,10 @@ sub get_latest_checkout_stations { push( @ret, { - name => $row->{arr_name}, - eva => $row->{arr_eva}, - hafas => ( $row->{train_id} =~ m{[|]} ? 'DB' : 0 ), + name => $row->{arr_name}, + eva => $row->{arr_eva}, + hafas => $row->{is_hafas} ? $row->{backend_name} : 0, + backend_id => $row->{backend_id}, } ); } @@ -1726,28 +1734,29 @@ sub get_stats { return $stats; } -sub get_latest_dest_id { +sub get_latest_dest_ids { my ( $self, %opt ) = @_; my $uid = $opt{uid}; my $db = $opt{db} // $self->{pg}->db; if ( - my $id = $self->{in_transit}->get_checkout_station_id( + my ( $id, $backend_id ) = $self->{in_transit}->get_checkout_ids( uid => $uid, db => $db ) ) { - return $id; + return ( $id, $backend_id ); } - return $self->get_latest_checkout_station_id( + return $self->get_latest_checkout_ids( uid => $uid, db => $db ); } +# Returns a listref of {eva, name} hashrefs for the specified backend. sub get_connection_targets { my ( $self, %opt ) = @_; @@ -1756,21 +1765,29 @@ sub get_connection_targets { // DateTime->now( time_zone => 'Europe/Berlin' )->subtract( months => 4 ); my $db = $opt{db} //= $self->{pg}->db; my $min_count = $opt{min_count} // 3; + my $dest_id = $opt{eva}; if ( $opt{destination_name} ) { - return ( - [], - [ { eva => $opt{eva}, name => $opt{destination_name} } ] - ); + return [ { eva => $opt{eva}, name => $opt{destination_name} } ]; } - my $dest_id = $opt{eva} // $self->get_latest_dest_id(%opt); + my $backend_id = $opt{backend_id}; + + if ( not $dest_id ) { + ( $dest_id, $backend_id ) = $self->get_latest_dest_ids(%opt); + } if ( not $dest_id ) { - return ( [], [] ); + return []; } - my $dest_ids = [ $dest_id, $self->{stations}->get_meta( eva => $dest_id ) ]; + my $dest_ids = [ + $dest_id, + $self->{stations}->get_meta( + eva => $dest_id, + backend_id => $backend_id, + ) + ]; my $res = $db->select( 'journeys', @@ -1778,7 +1795,8 @@ sub get_connection_targets { { user_id => $uid, checkin_station_id => $dest_ids, - real_departure => { '>', $threshold } + real_departure => { '>', $threshold }, + backend_id => $opt{backend_id}, }, { group_by => ['checkout_station_id'], @@ -1788,8 +1806,11 @@ sub get_connection_targets { my @destinations = $res->hashes->grep( sub { shift->{count} >= $min_count } ) ->map( sub { shift->{dest} } )->each; - @destinations = $self->{stations}->get_by_evas(@destinations); - return ( $dest_ids, \@destinations ); + @destinations = $self->{stations}->get_by_evas( + backend_id => $opt{backend_id}, + evas => [@destinations] + ); + return @destinations; } sub update_visibility { diff --git a/lib/Travelynx/Model/Stations.pm b/lib/Travelynx/Model/Stations.pm index ad219e0..7215aa5 100644 --- a/lib/Travelynx/Model/Stations.pm +++ b/lib/Travelynx/Model/Stations.pm @@ -14,38 +14,125 @@ sub new { return bless( \%opt, $class ); } -sub add_or_update { +sub get_backend_id { my ( $self, %opt ) = @_; - my $stop = $opt{stop}; - my $loc = $stop->loc; - my $source = 1; - my $db = $opt{db} // $self->{pg}->db; - - if ( my $s = $self->get_by_eva( $loc->eva, db => $db ) ) { - if ( $source == 1 and $s->{source} == 0 and not $s->{archived} ) { - return; + + if ( $opt{iris} ) { + + # special case + return 0; + } + if ( $opt{hafas} and $self->{backend_id}{hafas}{ $opt{hafas} } ) { + return $self->{backend_id}{hafas}{ $opt{hafas} }; + } + + my $db = $opt{db} // $self->{pg}->db; + my $backend_id = 0; + + if ( $opt{hafas} ) { + $backend_id = $db->select( + 'backends', + ['id'], + { + hafas => 1, + name => $opt{hafas} + } + )->hash->{id}; + $self->{backend_id}{hafas}{ $opt{hafas} } = $backend_id; + } + + return $backend_id; +} + +sub get_hafas_name { + my ( $self, %opt ) = @_; + + if ( exists $self->{hafas_name}{ $opt{backend_id} } ) { + return $self->{hafas_name}{ $opt{backend_id} }; + } + + my $db = $opt{db} // $self->{pg}->db; + my $hafas_name; + my $ret = $db->select( + 'backends', + ['name'], + { + hafas => 1, + id => $opt{backend_id}, } - $db->update( + )->hash; + + if ($ret) { + $hafas_name = $ret->{name}; + } + + $self->{hafas_name}{ $opt{backend_id} } = $hafas_name; + + return $hafas_name; +} + +sub get_backends { + my ( $self, %opt ) = @_; + + $opt{db} //= $self->{pg}->db; + + my $res = $opt{db}->select( 'backends', [ 'id', 'name', 'iris', 'hafas' ] ); + my @ret; + + while ( my $row = $res->hash ) { + push( + @ret, + { + id => $row->{id}, + name => $row->{name}, + iris => $row->{iris}, + hafas => $row->{hafas}, + } + ); + } + + return @ret; +} + +sub add_or_update { + my ( $self, %opt ) = @_; + my $stop = $opt{stop}; + my $loc = $stop->loc; + $opt{db} //= $self->{pg}->db; + + $opt{backend_id} //= $self->get_backend_id(%opt); + + if ( + my $s = $self->get_by_eva( + $loc->eva, + db => $opt{db}, + backend_id => $opt{backend_id} + ) + ) + { + $opt{db}->update( 'stations', { name => $loc->name, lat => $loc->lat, lon => $loc->lon, - source => $source, archived => 0 }, - { eva => $loc->eva } + { + eva => $loc->eva, + source => $opt{backend_id} + } ); return; } - $db->insert( + $opt{db}->insert( 'stations', { eva => $loc->eva, name => $loc->name, lat => $loc->lat, lon => $loc->lon, - source => $source, + source => $opt{backend_id}, archived => 0 } ); @@ -53,17 +140,20 @@ sub add_or_update { sub add_meta { my ( $self, %opt ) = @_; - my $db = $opt{db} // $self->{pg}->db; my $eva = $opt{eva}; my @meta = @{ $opt{meta} }; + $opt{db} //= $self->{pg}->db; + $opt{backend_id} //= $self->get_backend_id(%opt); + for my $meta (@meta) { if ( $meta != $eva ) { - $db->insert( + $opt{db}->insert( 'related_stations', { - eva => $eva, - meta => $meta + eva => $eva, + meta => $meta, + backend_id => $opt{backend_id}, }, { on_conflict => undef } ); @@ -82,7 +172,16 @@ sub get_meta { my $db = $opt{db} // $self->{pg}->db; my $eva = $opt{eva}; - my $res = $db->select( 'related_stations', ['meta'], { eva => $eva } ); + $opt{backend_id} //= $self->get_backend_id( %opt, db => $db ); + + my $res = $db->select( + 'related_stations', + ['meta'], + { + eva => $eva, + backend_id => $opt{backend_id} + } + ); my @ret; while ( my $row = $res->hash ) { @@ -93,9 +192,12 @@ sub get_meta { } sub get_for_autocomplete { - my ($self) = @_; + my ( $self, %opt ) = @_; + + $opt{backend_id} //= $self->get_backend_id(%opt); - my $res = $self->{pg}->db->select( 'stations', ['name'] ); + my $res = $self->{pg} + ->db->select( 'stations', ['name'], { source => $opt{backend_id} } ); my %ret; while ( my $row = $res->hash ) { @@ -113,18 +215,34 @@ sub get_by_eva { return; } - my $db = $opt{db} // $self->{pg}->db; + $opt{db} //= $self->{pg}->db; + $opt{backend_id} //= $self->get_backend_id(%opt); - return $db->select( 'stations', '*', { eva => $eva } )->hash; + return $opt{db}->select( + 'stations', + '*', + { + eva => $eva, + source => $opt{backend_id} + } + )->hash; } # Fast sub get_by_evas { - my ( $self, @evas ) = @_; + my ( $self, %opt ) = @_; - my @ret - = $self->{pg}->db->select( 'stations', '*', { eva => { '=', \@evas } } ) - ->hashes->each; + $opt{db} //= $self->{pg}->db; + $opt{backend_id} //= $self->get_backend_id(%opt); + + my @ret = $self->{pg}->db->select( + 'stations', + '*', + { + eva => { '=', $opt{evas} }, + source => $opt{backend_id} + } + )->hashes->each; return @ret; } @@ -132,10 +250,18 @@ sub get_by_evas { sub get_by_name { my ( $self, $name, %opt ) = @_; - my $db = $opt{db} // $self->{pg}->db; + $opt{db} //= $self->{pg}->db; + $opt{backend_id} //= $self->get_backend_id(%opt); - return $db->select( 'stations', '*', { name => $name }, { limit => 1 } ) - ->hash; + return $opt{db}->select( + 'stations', + '*', + { + name => $name, + source => $opt{backend_id} + }, + { limit => 1 } + )->hash; } # Slow @@ -152,16 +278,27 @@ sub get_by_names { sub get_by_ds100 { my ( $self, $ds100, %opt ) = @_; - my $db = $opt{db} // $self->{pg}->db; + $opt{db} //= $self->{pg}->db; + $opt{backend_id} //= $self->get_backend_id(%opt); - return $db->select( 'stations', '*', { ds100 => $ds100 }, { limit => 1 } ) - ->hash; + return $opt{db}->select( + 'stations', + '*', + { + ds100 => $ds100, + source => $opt{backend_id} + }, + { limit => 1 } + )->hash; } # Can be slow sub search { my ( $self, $identifier, %opt ) = @_; + $opt{db} //= $self->{pg}->db; + $opt{backend_id} //= $self->get_backend_id(%opt); + if ( $identifier =~ m{ ^ \d+ $ }x ) { return $self->get_by_eva( $identifier, %opt ) // $self->get_by_ds100( $identifier, %opt ) diff --git a/lib/Travelynx/Model/Traewelling.pm b/lib/Travelynx/Model/Traewelling.pm index 25648cc..c460b1a 100644 --- a/lib/Travelynx/Model/Traewelling.pm +++ b/lib/Travelynx/Model/Traewelling.pm @@ -224,6 +224,7 @@ sub get_pushable_accounts { join in_transit_str as i on t.user_id = i.user_id where t.push_sync = True and i.arr_eva is not null + and i.backend_id <= 1 and i.cancelled = False } ); diff --git a/lib/Travelynx/Model/Users.pm b/lib/Travelynx/Model/Users.pm index 4d90d92..7d3777b 100644 --- a/lib/Travelynx/Model/Users.pm +++ b/lib/Travelynx/Model/Users.pm @@ -205,6 +205,13 @@ sub get_privacy_by { return; } +sub set_backend { + my ( $self, %opt ) = @_; + $opt{db} //= $self->{pg}->db; + + $opt{db}->update('users', {backend_id => $opt{backend_id}}, {id => $opt{uid}}); +} + sub set_privacy { my ( $self, %opt ) = @_; my $db = $opt{db} // $self->{pg}->db; @@ -401,12 +408,13 @@ sub get { my $uid = $opt{uid}; my $user = $db->select( - 'users', + 'users_with_backend', 'id, name, status, public_level, email, ' . 'accept_follows, notifications, ' . 'extract(epoch from registered_at) as registered_at_ts, ' . 'extract(epoch from last_seen) as last_seen_ts, ' - . 'extract(epoch from deletion_requested) as deletion_requested_ts', + . 'extract(epoch from deletion_requested) as deletion_requested_ts, ' + . 'backend_id, backend_name, hafas', { id => $uid } )->hash; if ($user) { @@ -443,6 +451,9 @@ sub get { time_zone => 'Europe/Berlin' ) : undef, + backend_id => $user->{backend_id}, + backend_name => $user->{backend_name}, + backend_hafas => $user->{hafas}, }; } return undef; -- cgit v1.2.3