diff options
Diffstat (limited to 'lib/Travelynx/Command/work.pm')
-rw-r--r-- | lib/Travelynx/Command/work.pm | 829 |
1 files changed, 656 insertions, 173 deletions
diff --git a/lib/Travelynx/Command/work.pm b/lib/Travelynx/Command/work.pm index ac4416d..dc58a48 100644 --- a/lib/Travelynx/Command/work.pm +++ b/lib/Travelynx/Command/work.pm @@ -1,11 +1,14 @@ package Travelynx::Command::work; # Copyright (C) 2020-2023 Birte Kristina Friesel +# Copyright (C) 2025 networkException <git@nwex.de> # # SPDX-License-Identifier: AGPL-3.0-or-later use Mojo::Base 'Mojolicious::Command'; use Mojo::Promise; +use utf8; + use DateTime; use JSON; use List::Util; @@ -15,12 +18,17 @@ has description => 'Update real-time data of active journeys'; has usage => sub { shift->extract_usage }; sub run { - my ($self) = @_; + my ( $self, $backend ) = @_; my $now = DateTime->now( time_zone => 'Europe/Berlin' ); my $checkin_deadline = $now->clone->subtract( hours => 48 ); my $json = JSON->new; + if ( -e 'maintenance' ) { + $self->app->log->debug('work: "maintenance" file found, aborting'); + return; + } + my $num_incomplete = $self->app->in_transit->delete_incomplete_checkins( earlier_than => $checkin_deadline ); @@ -28,237 +36,699 @@ sub run { $self->app->log->debug("Removed ${num_incomplete} incomplete checkins"); } - my $errors = 0; + my $errors = 0; + my $backend_issues = 0; + my $rate_limit_counts = 0; + my $dbris_rate_limited = 0; for my $entry ( $self->app->in_transit->get_all_active ) { + if ( -e 'maintenance' ) { + $self->app->log->debug('work: "maintenance" file found, aborting'); + return; + } + my $uid = $entry->{user_id}; my $dep = $entry->{dep_eva}; my $arr = $entry->{arr_eva}; my $train_id = $entry->{train_id}; - if ( $train_id =~ m{[|]} ) { + if ( $train_id eq 'manual' ) { + if ( $arr + and $entry->{real_arr_ts} + and $now->epoch - $entry->{real_arr_ts} > 900 ) + { + $self->app->checkout_p( + station => $arr, + force => 2, + dep_eva => $dep, + arr_eva => $arr, + uid => $uid + )->wait; + } + } + + elsif ( $entry->{is_dbris} and ( not $backend or $backend eq 'dbris' ) ) + { + + eval { - $self->app->hafas->get_journey_p( trip_id => $train_id )->then( - sub { - my ($journey) = @_; + Mojo::Promise->timer( + $dbris_rate_limited ? 4.5 : ( $backend ? 2.0 : 1.0 ) ) + ->then( + sub { + return $self->app->dbris->get_journey_p( + trip_id => $train_id ); + } + )->then( + sub { + my ($journey) = @_; + + $dbris_rate_limited = 0; + + my $found_dep; + my $found_arr; + for my $stop ( $journey->route ) { + if ( $stop->eva == $dep ) { + $found_dep = $stop; + } + if ( $arr and $stop->eva == $arr ) { + $found_arr = $stop; + last; + } + } + if ( not $found_dep ) { + $self->app->log->debug( + "Did not find $dep within journey $train_id"); + return; + } - my $found_dep; - my $found_arr; - for my $stop ( $journey->route ) { - if ( $stop->eva == $dep ) { - $found_dep = $stop; + if ( $found_dep->rt_dep ) { + $self->app->in_transit->update_departure_dbris( + uid => $uid, + journey => $journey, + stop => $found_dep, + dep_eva => $dep, + arr_eva => $arr, + train_id => $train_id, + ); } - if ( $arr and $stop->eva == $arr ) { - $found_arr = $stop; - last; + if ( $found_dep->sched_dep + and $found_dep->dep->epoch > $now->epoch ) + { + $self->app->add_wagonorder( + uid => $uid, + train_id => $train_id, + is_departure => 1, + eva => $dep, + datetime => $found_dep->sched_dep, + train_type => $journey->type, + train_no => $journey->number, + ); + $self->app->add_stationinfo( $uid, 1, + $train_id, $found_dep->eva ); + } + + if ( + $found_arr + and + ( $found_arr->rt_arr or $found_arr->is_cancelled ) + ) + { + $self->app->in_transit->update_arrival_dbris( + uid => $uid, + journey => $journey, + train_id => $train_id, + stop => $found_arr, + dep_eva => $dep, + arr_eva => $arr + ); + } + if ( $found_arr and $found_arr->rt_arr ) { + if ( $found_arr->arr->epoch - $now->epoch < 600 ) { + $self->app->add_wagonorder( + uid => $uid, + train_id => $train_id, + is_arrival => 1, + eva => $arr, + datetime => $found_arr->sched_dep, + train_type => $journey->type, + train_no => $journey->number, + ); + $self->app->add_stationinfo( $uid, 0, + $train_id, $found_dep->eva, + $found_arr->eva ); + } + } + if ( $found_arr and $found_arr->is_cancelled ) { + + # check out (adds a cancelled journey and resets journey state + # to destination selection) + $self->app->checkout_p( + station => $arr, + force => 0, + dep_eva => $dep, + arr_eva => $arr, + uid => $uid + )->wait; } } - if ( not $found_dep ) { - return Mojo::Promise->reject( - "Did not find $dep within journey $train_id"); + )->catch( + sub { + my ($err) = @_; + $self->app->log->debug( +"work($uid) @ DBRIS $entry->{backend_name}: journey: $err" + ); + if ( $err =~ m{HTTP 429} ) { + $dbris_rate_limited = 1; + $rate_limit_counts += 1; + } + else { + $backend_issues += 1; + } } + )->wait; - if ( $found_dep->{rt_dep} ) { - $self->app->in_transit->update_departure_hafas( - uid => $uid, - journey => $journey, - stop => $found_dep, - dep_eva => $dep, - arr_eva => $arr + if ( $arr + and $entry->{real_arr_ts} + and $now->epoch - $entry->{real_arr_ts} > 900 ) + { + $self->app->checkout_p( + station => $arr, + force => 2, + dep_eva => $dep, + arr_eva => $arr, + uid => $uid + )->wait; + } + }; + if ($@) { + $errors += 1; + $self->app->log->error( + "work($uid) @ DBRIS $entry->{backend_name}: $@"); + } + } + + elsif ( $entry->{is_efa} and ( not $backend or $backend eq 'efa' ) ) { + eval { + $self->app->efa->get_journey_p( + trip_id => $train_id, + service => $entry->{backend_name} + )->then( + sub { + my ($journey) = @_; + + my $found_dep; + my $found_arr; + for my $stop ( $journey->route ) { + if ( $stop->id_num == $dep ) { + $found_dep = $stop; + } + if ( $arr and $stop->id_num == $arr ) { + $found_arr = $stop; + last; + } + } + if ( not $found_dep ) { + $self->app->log->debug( + "Did not find $dep within journey $train_id"); + return; + } + + if ( $found_dep->rt_dep ) { + $self->app->in_transit->update_departure_efa( + uid => $uid, + journey => $journey, + stop => $found_dep, + dep_eva => $dep, + arr_eva => $arr, + trip_id => $train_id, + ); + } + + if ( + $found_arr + and + ( $found_arr->rt_arr or $found_arr->is_cancelled ) + ) + { + $self->app->in_transit->update_arrival_efa( + uid => $uid, + journey => $journey, + stop => $found_arr, + dep_eva => $dep, + arr_eva => $arr, + trip_id => $train_id, + ); + } + if ( $found_arr and $found_arr->is_cancelled ) { + + # check out (adds a cancelled journey and resets journey state + # to destination selection) + $self->app->checkout_p( + station => $arr, + force => 0, + dep_eva => $dep, + arr_eva => $arr, + uid => $uid + )->wait; + } + } + )->catch( + sub { + my ($err) = @_; + $backend_issues += 1; + $self->app->log->error( +"work($uid) @ EFA $entry->{backend_name}: journey: $err" ); } + )->wait; - if ( $found_arr and $found_arr->{rt_arr} ) { - $self->app->in_transit->update_arrival_hafas( - uid => $uid, - journey => $journey, - stop => $found_arr, - dep_eva => $dep, - arr_eva => $arr + if ( $arr + and $entry->{real_arr_ts} + and $now->epoch - $entry->{real_arr_ts} > 900 ) + { + $self->app->checkout_p( + station => $arr, + force => 2, + dep_eva => $dep, + arr_eva => $arr, + uid => $uid + )->wait; + } + }; + if ($@) { + $errors += 1; + $self->app->log->error( + "work($uid) @ EFA $entry->{backend_name}: $@"); + } + } + + elsif ( $entry->{is_motis} and ( not $backend or $backend eq 'motis' ) ) + { + + eval { + $self->app->motis->get_trip_p( + service => $entry->{backend_name}, + trip_id => $train_id, + )->then( + sub { + my ($journey) = @_; + + for my $stopover ( $journey->stopovers ) { + if ( not defined $stopover->stop->{eva} ) { + + # Looks like MOTIS / transitous station IDs can change after the fact. + # So let's be safe rather than sorry, even if this causes way too many calls to the slow path + # (Stations::get_by_external_id uses string lookups). + # This function call implicitly sets $stopover->stop->{eva} for MOTIS backends. + $self->app->stations->add_or_update( + stop => $stopover->stop, + motis => $entry->{backend_name}, + ); + + $self->app->log->debug( "mapped " + . $stopover->stop->id . " to " + . $stopover->stop->{eva} ); + } + } + + my $found_departure; + my $found_arrival; + for my $stopover ( $journey->stopovers ) { + if ( $stopover->stop->{eva} == $dep ) { + $found_departure = $stopover; + } + + if ( $arr and $stopover->stop->{eva} == $arr ) { + $found_arrival = $stopover; + last; + } + } + + if ( not $found_departure ) { + $self->app->log->debug( + "Did not find $dep within trip $train_id"); + return; + } + + if ( $found_departure->realtime_departure ) { + $self->app->in_transit->update_departure_motis( + uid => $uid, + journey => $journey, + stopover => $found_departure, + dep_eva => $dep, + arr_eva => $arr, + train_id => $train_id, + ); + } + + if ( $found_arrival + and $found_arrival->realtime_arrival ) + { + $self->app->in_transit->update_arrival_motis( + uid => $uid, + journey => $journey, + train_id => $train_id, + stopover => $found_arrival, + dep_eva => $dep, + arr_eva => $arr + ); + } + } + )->catch( + sub { + my ($err) = @_; + $self->app->log->debug( +"work($uid) @ MOTIS $entry->{backend_name}: journey: $err" ); } + )->wait; + + if ( $arr + and $entry->{real_arr_ts} + and $now->epoch - $entry->{real_arr_ts} > 900 ) + { + $self->app->checkout_p( + station => $arr, + force => 2, + dep_eva => $dep, + arr_eva => $arr, + uid => $uid + )->wait; } - )->catch( - sub { - my ($err) = @_; - $self->app->log->error("work($uid)/journey: $err"); - } - )->wait; + }; + if ($@) { + $errors += 1; + $self->app->log->error( + "work($uid) @ MOTIS $entry->{backend_name}: $@"); + } + } - if ( $arr - and $entry->{real_arr_ts} - and $now->epoch - $entry->{real_arr_ts} > 600 ) - { - $self->app->checkout_p( - station => $arr, - force => 2, - dep_eva => $dep, - arr_eva => $arr, - uid => $uid + elsif ( $entry->{is_hafas} and ( not $backend or $backend eq 'hafas' ) ) + { + + eval { + + $self->app->hafas->get_journey_p( + trip_id => $train_id, + service => $entry->{backend_name} + )->then( + sub { + my ($journey) = @_; + + my $found_dep; + my $found_arr; + for my $stop ( $journey->route ) { + if ( $stop->loc->eva == $dep ) { + $found_dep = $stop; + } + if ( $arr and $stop->loc->eva == $arr ) { + $found_arr = $stop; + last; + } + } + if ( not $found_dep ) { + $self->app->log->debug( + "Did not find $dep within journey $train_id"); + return; + } + + if ( $found_dep->rt_dep ) { + $self->app->in_transit->update_departure_hafas( + uid => $uid, + journey => $journey, + stop => $found_dep, + dep_eva => $dep, + arr_eva => $arr + ); + } + if ( + $found_dep->sched_dep + and ( $entry->{backend_id} <= 1 + or $entry->{backend_name} eq 'VRN' + or $entry->{backend_name} eq 'ÖBB' ) + and $journey->class <= 16 + and $found_dep->dep->epoch > $now->epoch + ) + { + $self->app->add_wagonorder( + uid => $uid, + train_id => $journey->id, + is_departure => 1, + eva => $dep, + datetime => $found_dep->sched_dep, + train_type => ( $journey->type // q{} ) + =~ s{ +$}{}r, + train_no => $journey->number, + ); + $self->app->add_stationinfo( $uid, 1, + $journey->id, $found_dep->loc->eva ); + } + + if ( $found_arr and $found_arr->rt_arr ) { + $self->app->in_transit->update_arrival_hafas( + uid => $uid, + journey => $journey, + stop => $found_arr, + dep_eva => $dep, + arr_eva => $arr + ); + if ( + ( + $entry->{backend_id} <= 1 + or $entry->{backend_name} eq 'VRN' + or $entry->{backend_name} eq 'ÖBB' + ) + and $journey->class <= 16 + and $found_arr->arr->epoch - $now->epoch < 600 + ) + { + $self->app->add_wagonorder( + uid => $uid, + train_id => $journey->id, + is_arrival => 1, + eva => $arr, + datetime => $found_arr->sched_dep, + train_type => $journey->type, + train_no => $journey->number, + ); + $self->app->add_stationinfo( $uid, 0, + $journey->id, $found_dep->loc->eva, + $found_arr->loc->eva ); + } + } + } + )->catch( + sub { + my ($err) = @_; + $backend_issues += 1; + if ( $err + =~ m{svcResL\[0\][.]err is (?:FAIL|PARAMETER)$} + or $err =~ m{timeout} ) + { + # These are not actionable. + $self->app->log->debug( +"work($uid) @ HAFAS $entry->{backend_name}: journey: $err" + ); + } + else { + $self->app->log->error( +"work($uid) @ HAFAS $entry->{backend_name}: journey: $err" + ); + } + } )->wait; + + if ( $arr + and $entry->{real_arr_ts} + and $now->epoch - $entry->{real_arr_ts} > 900 ) + { + $self->app->checkout_p( + station => $arr, + force => 2, + dep_eva => $dep, + arr_eva => $arr, + uid => $uid + )->wait; + } + }; + if ($@) { + $errors += 1; + $self->app->log->error( + "work($uid) @ HAFAS $entry->{backend_name}: $@"); } - next; } + # TODO irgendwo ist hier ne race condition wo ein neuer checkin (in HAFAS) mit IRIS-Daten überschrieben wird. + # Die ganzen updates brauchen wirklich mal sanity checks mit train id ... + # Note: IRIS data is not always updated in real-time. Both departure and # arrival delays may take several minutes to appear, especially in case # of large-scale disturbances. We work around this by continuing to # update departure data for up to 15 minutes after departure and # delaying automatic checkout by at least 10 minutes. - eval { - if ( $now->epoch - $entry->{real_dep_ts} < 900 ) { - my $status = $self->app->iris->get_departures( - station => $dep, - lookbehind => 30, - lookahead => 30 - ); - if ( $status->{errstr} ) { - die("get_departures($dep): $status->{errstr}\n"); - } - - my ($train) = List::Util::first { $_->train_id eq $train_id } - @{ $status->{results} }; + elsif ( $entry->{is_iris} and ( not $backend or $backend eq 'iris' ) ) { + eval { + if ( $now->epoch - $entry->{real_dep_ts} < 900 ) { + my $status = $self->app->iris->get_departures( + station => $dep, + lookbehind => 30, + lookahead => 30 + ); + if ( $status->{errstr} ) { + die("get_departures($dep): $status->{errstr}\n"); + } - if ( not $train ) { - $self->app->log->info( - "could not find train $train_id at $dep\n"); - return; - } + my ($train) + = List::Util::first { $_->train_id eq $train_id } + @{ $status->{results} }; - $self->app->in_transit->update_departure( - uid => $uid, - train => $train, - dep_eva => $dep, - arr_eva => $arr, - route => [ $self->app->iris->route_diff($train) ] - ); + if ( not $train ) { + $self->app->log->debug( + "could not find train $train_id at $dep\n"); + return; + } - if ( $train->departure_is_cancelled and $arr ) { - my $checked_in - = $self->app->in_transit->update_departure_cancelled( + $self->app->in_transit->update_departure( uid => $uid, train => $train, dep_eva => $dep, arr_eva => $arr, - ); - - # depending on the amount of users in transit, some time may - # have passed between fetching $entry from the database and - # now. Only check out if the user is still checked into this - # train. - if ($checked_in) { + route => [ $self->app->iris->route_diff($train) ] + ); - # check out (adds a cancelled journey and resets journey state - # to checkin - $self->app->checkout_p( - station => $arr, - force => 2, + if ( $train->departure_is_cancelled and $arr ) { + my $checked_in + = $self->app->in_transit->update_departure_cancelled( + uid => $uid, + train => $train, dep_eva => $dep, arr_eva => $arr, - uid => $uid - )->wait; + ); + + # depending on the amount of users in transit, some time may + # have passed between fetching $entry from the database and + # now. Only check out if the user is still checked into this + # train. + if ($checked_in) { + + # check out (adds a cancelled journey and resets journey state + # to checkin + $self->app->checkout_p( + station => $arr, + force => 2, + dep_eva => $dep, + arr_eva => $arr, + uid => $uid + )->wait; + } + } + else { + $self->app->add_route_timestamps( $uid, $train, 1 ); + $self->app->add_wagonorder( + uid => $uid, + train_id => $train->train_id, + is_departure => 1, + eva => $dep, + datetime => $train->sched_departure, + train_type => $train->type, + train_no => $train->train_no + ); + $self->app->add_stationinfo( $uid, 1, $train->train_id, + $dep, $arr ); } } - else { - $self->app->add_route_timestamps( $uid, $train, 1 ); - } + }; + if ($@) { + $errors += 1; + $self->app->log->error("work($uid) @ IRIS: departure: $@"); } - }; - if ($@) { - $errors += 1; - $self->app->log->error("work($uid)/departure: $@"); - } - eval { - if ( - $arr - and ( not $entry->{real_arr_ts} - or $now->epoch - $entry->{real_arr_ts} < 600 ) - ) - { - my $status = $self->app->iris->get_departures( - station => $arr, - lookbehind => 20, - lookahead => 220 - ); - if ( $status->{errstr} ) { - die("get_departures($arr): $status->{errstr}\n"); - } + eval { + if ( + $arr + and ( not $entry->{real_arr_ts} + or $now->epoch - $entry->{real_arr_ts} < 600 ) + ) + { + my $status = $self->app->iris->get_departures( + station => $arr, + lookbehind => 20, + lookahead => 220 + ); + if ( $status->{errstr} ) { + die("get_departures($arr): $status->{errstr}\n"); + } - # Note that a train may pass the same station several times. - # Notable example: S41 / S42 ("Ringbahn") both starts and - # terminates at Berlin Südkreuz - my ($train) = List::Util::first { - $_->train_id eq $train_id - and $_->sched_arrival - and $_->sched_arrival->epoch > $entry->{sched_dep_ts} - } - @{ $status->{results} }; + # Note that a train may pass the same station several times. + # Notable example: S41 / S42 ("Ringbahn") both starts and + # terminates at Berlin Südkreuz + my ($train) = List::Util::first { + $_->train_id eq $train_id + and $_->sched_arrival + and $_->sched_arrival->epoch > $entry->{sched_dep_ts} + } + @{ $status->{results} }; - $train //= List::Util::first { $_->train_id eq $train_id } - @{ $status->{results} }; + $train //= List::Util::first { $_->train_id eq $train_id } + @{ $status->{results} }; - if ( not $train ) { + if ( not $train ) { - # If we haven't seen the train yet, its arrival is probably - # too far in the future. This is not critical. - return; - } + # If we haven't seen the train yet, its arrival is probably + # too far in the future. This is not critical. + return; + } - my $checked_in = $self->app->in_transit->update_arrival( - uid => $uid, - train => $train, - route => [ $self->app->iris->route_diff($train) ], - dep_eva => $dep, - arr_eva => $arr, - ); + my $checked_in = $self->app->in_transit->update_arrival( + uid => $uid, + train => $train, + route => [ $self->app->iris->route_diff($train) ], + dep_eva => $dep, + arr_eva => $arr, + ); - if ( $checked_in and $train->arrival_is_cancelled ) { + if ( $checked_in and $train->arrival_is_cancelled ) { - # check out (adds a cancelled journey and resets journey state - # to destination selection) - $self->app->checkout_p( + # check out (adds a cancelled journey and resets journey state + # to destination selection) + $self->app->checkout_p( + station => $arr, + force => 0, + dep_eva => $dep, + arr_eva => $arr, + uid => $uid + )->wait; + } + else { + $self->app->add_route_timestamps( + $uid, $train, 0, + ( + defined $entry->{real_arr_ts} + and $now->epoch > $entry->{real_arr_ts} + ) ? 1 : 0 + ); + $self->app->add_wagonorder( + uid => $uid, + train_id => $train->train_id, + is_arrival => 1, + eva => $arr, + datetime => $train->sched_departure, + train_type => $train->type, + train_no => $train->train_no + ); + $self->app->add_stationinfo( $uid, 0, $train->train_id, + $dep, $arr ); + } + } + elsif ( $entry->{real_arr_ts} ) { + my ( undef, $error ) = $self->app->checkout_p( station => $arr, - force => 0, + force => 2, dep_eva => $dep, arr_eva => $arr, uid => $uid + )->catch( + sub { + my ($error) = @_; + $backend_issues += 1; + $self->app->log->error( + "work($uid) @ IRIS: arrival: $error"); + $errors += 1; + } )->wait; } - else { - $self->app->add_route_timestamps( - $uid, $train, 0, - ( - defined $entry->{real_arr_ts} - and $now->epoch > $entry->{real_arr_ts} - ) ? 1 : 0 - ); - } - } - elsif ( $entry->{real_arr_ts} ) { - my ( undef, $error ) = $self->app->checkout_p( - station => $arr, - force => 2, - dep_eva => $dep, - arr_eva => $arr, - uid => $uid - )->catch( - sub { - my ($error) = @_; - $self->app->log->error("work($uid)/arrival: $error"); - $errors += 1; - } - )->wait; + }; + if ($@) { + $self->app->log->error("work($uid) @ IRIS: arrival: $@"); + $errors += 1; } - }; - if ($@) { - $self->app->log->error("work($uid)/arrival: $@"); - $errors += 1; + + eval { }; } - eval { }; } my $started_at = $now; @@ -266,22 +736,35 @@ sub run { my $worker_duration = $main_finished_at->epoch - $started_at->epoch; if ( $self->app->config->{influxdb}->{url} ) { + my $tags = q{}; + if ($backend) { + $tags .= ",backend=${backend}"; + } if ( $self->app->mode eq 'development' ) { $self->app->log->debug( 'POST ' . $self->app->config->{influxdb}->{url} - . " worker runtime_seconds=${worker_duration},errors=${errors}" + . " worker${tags} runtime_seconds=${worker_duration},errors=${errors},backend_errors=${backend_issues},ratelimit_count=${rate_limit_counts}" ); } else { $self->app->ua->post_p( $self->app->config->{influxdb}->{url}, - "worker runtime_seconds=${worker_duration},errors=${errors}" ) - ->wait; +"worker${tags} runtime_seconds=${worker_duration},errors=${errors},backend_errors=${backend_issues},ratelimit_count=${rate_limit_counts}" + )->wait; } } if ( not $self->app->config->{traewelling}->{separate_worker} ) { $self->app->start('traewelling'); } + + # add_wagonorder and add_stationinfo assume a permanently running IOLoop + # and do not allow Mojolicious commands to wait until they have completed. + # Hence, some add_wagonorder and add_stationinfo calls made here may not + # complete before the work command exits, and thus have no effect. + # + # This is not ideal and will need fixing at some point. Until then, here + # is the pragmatic solution for 99% of the associated issues. + Mojo::Promise->timer(5)->wait; } 1; |