diff --git a/src/main/java/org/mtransit/parser/DefaultAgencyTools.java b/src/main/java/org/mtransit/parser/DefaultAgencyTools.java index ddf15fe..db6011a 100644 --- a/src/main/java/org/mtransit/parser/DefaultAgencyTools.java +++ b/src/main/java/org/mtransit/parser/DefaultAgencyTools.java @@ -41,6 +41,8 @@ import org.mtransit.parser.mt.data.MDirection; import org.mtransit.parser.mt.data.MString; import org.mtransit.parser.mt.data.MStrings; +import org.mtransit.parser.mt.data.MTripId; +import org.mtransit.parser.mt.data.MTripIds; import org.mtransit.parser.mt.data.MVerify; import java.text.SimpleDateFormat; @@ -73,19 +75,13 @@ public class DefaultAgencyTools implements GAgencyTools { private static final int MAX_LOOK_BACKWARD_IN_DAYS = 10 * 365; // used for CURRENT schedule from calendar_dates.txt all in the past private static final int MAX_LOOK_FORWARD_IN_DAYS = 60; - private static final int MIN_CALENDAR_COVERAGE_TOTAL_IN_DAYS = 5; + private static final int MIN_CALENDAR_COVERAGE_TOTAL_IN_DAYS = 5; // = 6 days private static final int MIN_CALENDAR_DATE_COVERAGE_TOTAL_IN_DAYS = 14; // 2024-04-09: 10x -> 3x because merging 2 schedule can create very bad schedule info #GRTbus private static final long MAX_CALENDAR_DATE_COVERAGE_RATIO = 3; - private static final int MIN_PREVIOUS_NEXT_ADDED_DAYS = 2; - - public static final boolean EXPORT_TRIP_ID; - - static { - EXPORT_TRIP_ID = false; - } + private static final int MIN_PREVIOUS_NEXT_ADDED_DAYS = 2; // = 1 day @SuppressWarnings("WeakerAccess") public static final boolean GOOD_ENOUGH_ACCEPTED; @@ -155,9 +151,11 @@ public void start(@NotNull String[] args) { MTLog.log("Generating data..."); MTLog.logDebug("Args [%d]: %s.", args.length, Arrays.asList(args)); final List lastServiceDates = MReader.loadServiceDates(args[2]); + final List lastTripIds = MReader.loadTripIds(args[2]); final List lastServiceIds = MReader.loadServiceIds(args[2]); final List lastStrings = MReader.loadStrings(args[2]); MServiceIds.addAll(lastServiceIds); + MTripIds.addAll(lastTripIds); MStrings.addAll(lastStrings); this.serviceIdInts = extractUsefulServiceIdInts(args, this, true, lastServiceDates); final String inputUrl = args.length >= 5 ? args[4] : null; @@ -1319,7 +1317,7 @@ public static HashSet extractUsefulServiceIdInts( final List gCalendars = gtfs.getAllCalendars(); final List gCalendarDates = gtfs.getAllCalendarDates(); final Period entirePeriod = getEntirePeriodMinMaxDate(gCalendars, gCalendarDates); - MTLog.log("* Schedule available from %s to %s.", entirePeriod.getStartDate(), entirePeriod.getEndDate()); + MTLog.log("* Entire schedule available: from %s to %s.", entirePeriod.getStartDate(), entirePeriod.getEndDate()); MTLog.log("------------------------------"); MTLog.log("* Looking for CURRENT schedules..."); boolean hasCurrent = false; @@ -1577,12 +1575,12 @@ && diffLowerThan(DATE_FORMAT, c, p.getTodayStringInt(), initialTodayStringInt, M static void parseCalendars(@NotNull List gCalendars, @Nullable List gCalendarDates, SimpleDateFormat DATE_FORMAT, Calendar c, Period p, boolean lookBackward) { findCalendarsTodayPeriod(gCalendars, gCalendarDates, p, lookBackward); if (p.getStartDate() == null || p.getEndDate() == null) { - MTLog.log("NO schedule available for %s in calendars. (start:%s|end:%s)", p.getTodayStringInt(), p.getStartDate(), p.getEndDate()); + MTLog.log("[parse-calendars] > NO schedule available for %s in calendars. (start:%s|end:%s)", p.getTodayStringInt(), p.getStartDate(), p.getEndDate()); return; } boolean newDates; while (true) { - MTLog.log("Schedules from %s to %s... ", p.getStartDate(), p.getEndDate()); + MTLog.log("[parse-calendars] > Schedules from '%s' to '%s'... ", p.getStartDate(), p.getEndDate()); newDates = false; for (GCalendar gCalendar : gCalendars) { if (!gCalendar.isOverlapping(p.getStartDate(), p.getEndDate())) { @@ -1590,16 +1588,18 @@ static void parseCalendars(@NotNull List gCalendars, @Nullable List ignored service ID from calendar date active between %s and %s: %s (SERVICE REMOVED)", gCalendar.getStartDate(), gCalendar.getEndDate(), gCalendar.getServiceId()); + MTLog.logDebug("[parse-calendars] > ignored service ID from calendar date active between %s and %s: %s (SERVICE REMOVED)", gCalendar.getStartDate(), gCalendar.getEndDate(), gCalendar.getServiceId()); continue; } if (p.getStartDate() == null || gCalendar.startsBefore(p.getStartDate())) { - MTLog.log("new start date from calendar active between %s and %s: %s (was: %s)", p.getStartDate(), p.getEndDate(), gCalendar.getStartDate(), p.getStartDate()); + //noinspection DiscouragedApi + MTLog.log("[parse-calendars] > (today: %s) new start date '%s' from calendar (service:'%s'|start:%s|end:%s) active (was: %s)", p.getTodayStringInt(), gCalendar.getStartDate(), gCalendar.getServiceId(), gCalendar.getStartDate(), gCalendar.getEndDate(), p.getStartDate()); p.setStartDate(gCalendar.getStartDate()); newDates = true; } if (p.getEndDate() == null || gCalendar.endsAfter(p.getEndDate())) { - MTLog.log("new end date from calendar active between %s and %s: %s (was: %s)", p.getStartDate(), p.getEndDate(), gCalendar.getEndDate(), p.getEndDate()); + //noinspection DiscouragedApi + MTLog.log("[parse-calendars] > (today: %s) new end date '%s' from calendar (service:'%s'|start:%s|end:%s) active (was: %s)", p.getTodayStringInt(), gCalendar.getEndDate(), gCalendar.getServiceId(), gCalendar.getStartDate(), gCalendar.getEndDate(), p.getEndDate()); p.setEndDate(gCalendar.getEndDate()); newDates = true; } @@ -1607,31 +1607,37 @@ static void parseCalendars(@NotNull List gCalendars, @Nullable List new end date '%s' because next day has own service ID(s)", p.getEndDate()); continue; + } else if (pNext.getStartDate() != null && pNext.getEndDate() != null) { + MTLog.logDebug("[parse-calendars] > ignore next period because coverage '%s' not < %s days: %s", + MTLog.formatDuration(diffInMs(DATE_FORMAT, c, pNext.getStartDate(), pNext.getEndDate())), + MIN_PREVIOUS_NEXT_ADDED_DAYS, pNext.getTodayStringInt()); } - Period pPrevious = new Period(); - pPrevious.setTodayStringInt(incDateDays(DATE_FORMAT, c, p.getStartDate(), -1)); - findDayServiceIdsPeriod(gCalendars, gCalendarDates, pPrevious); if (diffLowerThan(DATE_FORMAT, c, p.getStartDate(), p.getEndDate(), MIN_CALENDAR_COVERAGE_TOTAL_IN_DAYS)) { - long nextPeriodCoverageInMs = pNext.getStartDate() == null || pNext.getEndDate() == null ? 0L : diffInMs(DATE_FORMAT, c, pNext.getStartDate(), pNext.getEndDate()); - long previousPeriodCoverageInMs = pPrevious.getStartDate() == null || pPrevious.getEndDate() == null ? 0L : diffInMs(DATE_FORMAT, c, pPrevious.getStartDate(), pPrevious.getEndDate()); - if (lookBackward // NOT next schedule, only current schedule can look behind - && previousPeriodCoverageInMs > 0L && previousPeriodCoverageInMs < nextPeriodCoverageInMs) { - p.setStartDate(incDateDays(DATE_FORMAT, c, p.getStartDate(), -1)); // start-- - MTLog.log("new start date because coverage lower than %s days: %s", MIN_CALENDAR_COVERAGE_TOTAL_IN_DAYS, p.getStartDate()); - } else { - p.setEndDate(incDateDays(DATE_FORMAT, c, p.getEndDate(), 1)); // end++ - MTLog.log("new end date because coverage lower than %s days: %s", MIN_CALENDAR_COVERAGE_TOTAL_IN_DAYS, p.getEndDate()); + if (lookBackward) { // NOT next schedule, only current schedule can look behind + final Period pPrevious = new Period(); + pPrevious.setTodayStringInt(incDateDays(DATE_FORMAT, c, p.getStartDate(), -1)); + findDayServiceIdsPeriod(gCalendars, gCalendarDates, pPrevious); + long nextPeriodCoverageInMs = pNext.getStartDate() == null || pNext.getEndDate() == null ? 0L : diffInMs(DATE_FORMAT, c, pNext.getStartDate(), pNext.getEndDate()); + long previousPeriodCoverageInMs = pPrevious.getStartDate() == null || pPrevious.getEndDate() == null ? 0L : diffInMs(DATE_FORMAT, c, pPrevious.getStartDate(), pPrevious.getEndDate()); + if (previousPeriodCoverageInMs > 0L && previousPeriodCoverageInMs < nextPeriodCoverageInMs) { + p.setStartDate(incDateDays(DATE_FORMAT, c, p.getStartDate(), -1)); // start-- + MTLog.log("[parse-calendars] > new start date because coverage lower than %s days: %s", MIN_CALENDAR_COVERAGE_TOTAL_IN_DAYS, p.getStartDate()); + continue; + } } + p.setEndDate(incDateDays(DATE_FORMAT, c, p.getEndDate(), 1)); // end++ + MTLog.log("[parse-calendars] > new end date because coverage not < %s days: %s", MIN_CALENDAR_COVERAGE_TOTAL_IN_DAYS, p.getEndDate()); continue; } + MTLog.logDebug("[parse-calendars] > stop here with good coverage from '%s' to '%s' (diff: %s)", p.getStartDate(), p.getEndDate(), MTLog.formatDuration(diffInMs(DATE_FORMAT, c, p.getStartDate(), p.getEndDate()))); break; } } @@ -1642,19 +1648,23 @@ static void findCalendarsTodayPeriod(List gCalendars, List (today:%s) SKIP outside service ID '%s' from calendar date active between '%s' and '%s'", p.getTodayStringInt(), gCalendar.getServiceId(), gCalendar.getStartDate(), gCalendar.getEndDate()); continue; } if (GCalendarDate.isServiceEntirelyRemoved(gCalendar, gCalendarDates)) { //noinspection DiscouragedApi - MTLog.logDebug("findCalendarsTodayPeriod() > ignored service ID from calendar date active between %s and %s: %s (SERVICE REMOVED)", gCalendar.getStartDate(), gCalendar.getEndDate(), gCalendar.getServiceId()); + // MTLog.logDebug("[find-today-period] > (today:%s) SKIP removed service ID '%s' from calendar date active between '%s' and '%s'", p.getTodayStringInt(), gCalendar.getServiceId(), gCalendar.getStartDate(), gCalendar.getEndDate()); continue; } if (p.getStartDate() == null || gCalendar.startsBefore(p.getStartDate())) { - MTLog.log("new start date from calendar active on %s: %s (was: %s)", p.getTodayStringInt(), gCalendar.getStartDate(), p.getStartDate()); + //noinspection DiscouragedApi + MTLog.log("[find-today-period] > (today:%s) new start date '%s' from active calendar (service:'%s') (was: %s)", p.getTodayStringInt(), gCalendar.getStartDate(), gCalendar.getServiceId(), p.getStartDate()); p.setStartDate(gCalendar.getStartDate()); } if (p.getEndDate() == null || gCalendar.endsAfter(p.getEndDate())) { - MTLog.log("new end date from calendar active on %s: %s (was: %s)", p.getTodayStringInt(), gCalendar.getEndDate(), p.getEndDate()); + //noinspection DiscouragedApi + MTLog.log("[find-today-period] > (today:%s) new end date '%s' from active calendar (service:'%s') (was: %s)", p.getTodayStringInt(), gCalendar.getEndDate(), gCalendar.getServiceId(), p.getEndDate()); p.setEndDate(gCalendar.getEndDate()); } } @@ -1663,14 +1673,14 @@ static void findCalendarsTodayPeriod(List gCalendars, List earlier today because no service: %s (initial today: %s)", p.getTodayStringInt(), initialTodayStringInt); continue; } } else { // #NEXT if (entirePeriod.getStartDate() != null && p.getTodayStringInt() != null && entirePeriod.getStartDate() > p.getTodayStringInt()) { p.setTodayStringInt(entirePeriod.getStartDate()); - MTLog.log("latter today because no service: %s (initial today: %s)", p.getTodayStringInt(), initialTodayStringInt); + MTLog.log("[find-today-period] > latter today because no service: %s (initial today: %s)", p.getTodayStringInt(), initialTodayStringInt); continue; } } @@ -1680,6 +1690,7 @@ static void findCalendarsTodayPeriod(List gCalendars, List gCalendars, @Nullable List gCalendarDates, Period p) { + final Set serviceIdInts = new HashSet<>(); boolean newDates; while (true) { newDates = false; @@ -1689,17 +1700,19 @@ static void findDayServiceIdsPeriod(List gCalendars, @Nullable List ignored service ID from calendar date active between %s and %s: %s (SERVICE REMOVED)", gCalendar.getStartDate(), gCalendar.getEndDate(), gCalendar.getServiceId()); + logFindDayServiceIdsPeriod("[find-day-service-id-period] > ignored service ID from calendar date active between %s and %s: %s (SERVICE REMOVED)", gCalendar.getStartDate(), gCalendar.getEndDate(), gCalendar.getServiceId()); continue; } if (p.getStartDate() == null || gCalendar.startsBefore(p.getStartDate())) { - MTLog.logDebug("findDayServiceIdsPeriod() > new start date from calendar active on %s: %s (was: %s)", p.getTodayStringInt(), gCalendar.getStartDate(), p.getStartDate()); + logFindDayServiceIdsPeriod("[find-day-service-id-period] > new start date from calendar active on %s: %s (was: %s)", p.getTodayStringInt(), gCalendar.getStartDate(), p.getStartDate()); p.setStartDate(gCalendar.getStartDate()); + serviceIdInts.add(gCalendar.getServiceIdInt()); newDates = true; } if (p.getEndDate() == null || gCalendar.endsAfter(p.getEndDate())) { - MTLog.logDebug("findDayServiceIdsPeriod() > new end date from calendar active on %s: %s (was: %s)", p.getTodayStringInt(), gCalendar.getEndDate(), p.getEndDate()); + logFindDayServiceIdsPeriod("[find-day-service-id-period] > new end date from calendar active on %s: %s (was: %s)", p.getTodayStringInt(), gCalendar.getEndDate(), p.getEndDate()); p.setEndDate(gCalendar.getEndDate()); + serviceIdInts.add(gCalendar.getServiceIdInt()); newDates = true; } } @@ -1709,11 +1722,11 @@ static void findDayServiceIdsPeriod(List gCalendars, @Nullable List NO schedule available for %s in calendars. (start:%s|end:%s)", p.getTodayStringInt(), p.getStartDate(), p.getEndDate()); + logFindDayServiceIdsPeriod("[find-day-service-id-period] > NO schedule available for %s in calendars. (start:%s|end:%s)", p.getTodayStringInt(), p.getStartDate(), p.getEndDate()); return; } while (true) { - MTLog.logDebug("findDayServiceIdsPeriod() > Schedules from %s to %s... ", p.getStartDate(), p.getEndDate()); + logFindDayServiceIdsPeriod("[find-day-service-id-period] > Schedules from %s to %s... ", p.getStartDate(), p.getEndDate()); newDates = false; for (GCalendar gCalendar : gCalendars) { if (!gCalendar.isOverlapping(p.getStartDate(), p.getEndDate())) { @@ -1721,17 +1734,19 @@ static void findDayServiceIdsPeriod(List gCalendars, @Nullable List ignored service ID from calendar date active between %s and %s: %s (SERVICE REMOVED)", gCalendar.getStartDate(), gCalendar.getEndDate(), gCalendar.getServiceId()); + logFindDayServiceIdsPeriod("[find-day-service-id-period] > ignored service ID from calendar date active between %s and %s: %s (SERVICE REMOVED)", gCalendar.getStartDate(), gCalendar.getEndDate(), gCalendar.getServiceId()); continue; } if (p.getStartDate() == null || gCalendar.startsBefore(p.getStartDate())) { - MTLog.logDebug("findDayServiceIdsPeriod() > new start date from calendar active between %s and %s: %s (was: %s)", p.getStartDate(), p.getEndDate(), gCalendar.getStartDate(), p.getStartDate()); + logFindDayServiceIdsPeriod("[find-day-service-id-period] > new start date from calendar active between %s and %s: %s (was: %s)", p.getStartDate(), p.getEndDate(), gCalendar.getStartDate(), p.getStartDate()); p.setStartDate(gCalendar.getStartDate()); + serviceIdInts.add(gCalendar.getServiceIdInt()); newDates = true; } if (p.getEndDate() == null || gCalendar.endsAfter(p.getEndDate())) { - MTLog.logDebug("findDayServiceIdsPeriod() > new end date from calendar active between %s and %s: %s (was: %s)", p.getStartDate(), p.getEndDate(), gCalendar.getEndDate(), p.getEndDate()); + logFindDayServiceIdsPeriod("[find-day-service-id-period] > new end date from calendar active between %s and %s: %s (was: %s)", p.getStartDate(), p.getEndDate(), gCalendar.getEndDate(), p.getEndDate()); p.setEndDate(gCalendar.getEndDate()); + serviceIdInts.add(gCalendar.getServiceIdInt()); newDates = true; } } @@ -1740,6 +1755,13 @@ static void findDayServiceIdsPeriod(List gCalendars, @Nullable List schedule available for '%s' in calendars. (start:%s|end:%s) service IDs:[%s].", p.getTodayStringInt(), p.getStartDate(), p.getEndDate(), GIDs.toStringPlus(serviceIdInts)); + } + + private static void logFindDayServiceIdsPeriod(@NotNull String format, @Nullable Object... args) { + //noinspection ConstantValue + if (true) return; // DEBUG + MTLog.logDebug(format, args); } @NotNull @@ -1757,11 +1779,11 @@ public static HashSet getPeriodServiceIds( } if (GCalendarDate.isServiceEntirelyRemoved(gCalendar, gCalendarDates, startDate, endDate)) { //noinspection DiscouragedApi - MTLog.logDebug("getPeriodServiceIds() > ignored service ID from calendar date active between %s and %s: %s (SERVICE REMOVED)", startDate, endDate, gCalendar.getServiceId()); + MTLog.logDebug("[period-service-ids] > ignored service ID from calendar date active between %s and %s: %s (SERVICE REMOVED)", startDate, endDate, gCalendar.getServiceId()); continue; } //noinspection DiscouragedApi - MTLog.log("new service ID from calendar active between %s and %s: %s", startDate, endDate, gCalendar.getServiceId()); + MTLog.log("[period-service-ids] > new service ID from calendar active between %s and %s: %s", startDate, endDate, gCalendar.getServiceId()); serviceIdInts.add(gCalendar.getServiceIdInt()); } } @@ -1774,16 +1796,16 @@ public static HashSet getPeriodServiceIds( } if (gCalendarDate.getExceptionType() == GCalendarDatesExceptionType.SERVICE_REMOVED) { //noinspection DiscouragedApi - MTLog.log("ignored service ID from calendar date active between %s and %s: %s (SERVICE REMOVED)", startDate, endDate, gCalendarDate.getServiceId()); + MTLog.log("[period-service-ids] > ignored service ID from calendar date active between %s and %s: %s (SERVICE REMOVED)", startDate, endDate, gCalendarDate.getServiceId()); continue; } //noinspection DiscouragedApi - MTLog.log("new service ID from calendar date active between %s and %s: %s", startDate, endDate, gCalendarDate.getServiceId()); + MTLog.log("[period-service-ids] > new service ID from calendar date active between %s and %s: %s", startDate, endDate, gCalendarDate.getServiceId()); serviceIdInts.add(gCalendarDate.getServiceIdInt()); } } } - MTLog.log("Service IDs [%d]: %s.", serviceIdInts.size(), GIDs.toStringPlus(CollectionUtils.sorted(serviceIdInts))); + MTLog.log("[period-service-ids] > Service IDs [%d]: %s.", serviceIdInts.size(), GIDs.toStringPlus(CollectionUtils.sorted(serviceIdInts))); return serviceIdInts; } diff --git a/src/main/java/org/mtransit/parser/TimeUtils.java b/src/main/java/org/mtransit/parser/TimeUtils.java index 8e1d847..4ab6252 100644 --- a/src/main/java/org/mtransit/parser/TimeUtils.java +++ b/src/main/java/org/mtransit/parser/TimeUtils.java @@ -2,16 +2,18 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.mtransit.commons.FeatureFlags; import java.util.Locale; @SuppressWarnings("WeakerAccess") public final class TimeUtils { - private static final int PRECISION_IN_SECONDS = 10; + private static final int PRECISION_IN_SECONDS = FeatureFlags.F_SCHEDULE_IN_MINUTES ? 1 : 10; @Nullable public static Integer cleanExtraSeconds(@Nullable Integer time) { + if (PRECISION_IN_SECONDS <= 1) return time; int extraSeconds = time == null ? 0 : time % PRECISION_IN_SECONDS; if (extraSeconds > 0) { // IF too precise DO return cleanTime(time, extraSeconds); diff --git a/src/main/java/org/mtransit/parser/TimeUtilsExt.kt b/src/main/java/org/mtransit/parser/TimeUtilsExt.kt index 2646047..edcd7ea 100644 --- a/src/main/java/org/mtransit/parser/TimeUtilsExt.kt +++ b/src/main/java/org/mtransit/parser/TimeUtilsExt.kt @@ -5,17 +5,21 @@ import java.time.ZoneId import java.time.format.DateTimeFormatter import java.util.Date import java.util.Locale +import kotlin.math.abs import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.nanoseconds fun formatSimpleDuration(durationInMs: Long) = buildString { - durationInMs.milliseconds.toComponents { days, hours, minutes, seconds, nanoseconds -> - days.takeIf { it > 0 }?.let { append(days).append("d ") } - hours.takeIf { it > 0 }?.let { append(hours).append("h ") } - minutes.takeIf { it > 0 }?.let { append(minutes).append("m ") } - seconds.takeIf { it > 0 }?.let { append(seconds).append("s ") } - nanoseconds.takeIf { it > 0 }?.let { append(nanoseconds).append("ns ") } + val negative = durationInMs < 0 + abs(durationInMs).milliseconds.toComponents { days, hours, minutes, seconds, nanoseconds -> + days.takeIf { it > 0 }?.let { append(it).append("d ") } + hours.takeIf { it > 0 }?.let { append(it).append("h ") } + minutes.takeIf { it > 0 }?.let { append(it).append("m ") } + seconds.takeIf { it > 0 }?.let { append(it).append("s ") } + nanoseconds.takeIf { it > 0 }?.nanoseconds?.inWholeMilliseconds?.let { append(it).append("ms ") } } -} + if (negative) insert(0, "-") +}.trim() private val shortDateTimeFormatter = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss z", Locale.ENGLISH) .withZone(ZoneId.systemDefault()) diff --git a/src/main/java/org/mtransit/parser/db/DumpDbUtils.kt b/src/main/java/org/mtransit/parser/db/DumpDbUtils.kt index acac9ca..b9f1864 100644 --- a/src/main/java/org/mtransit/parser/db/DumpDbUtils.kt +++ b/src/main/java/org/mtransit/parser/db/DumpDbUtils.kt @@ -35,6 +35,9 @@ object DumpDbUtils { if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) { SQLUtils.executeUpdate(statement, GTFSCommons.T_SERVICE_IDS_SQL_DROP) } + if (FeatureFlags.F_EXPORT_TRIP_ID_INTS) { + SQLUtils.executeUpdate(statement, GTFSCommons.T_TRIP_IDS_SQL_DROP) + } SQLUtils.executeUpdate(statement, GTFSCommons.T_SERVICE_DATES_SQL_DROP) // CREATE SQLUtils.executeUpdate(statement, GTFSCommons.T_ROUTE_SQL_CREATE) @@ -44,6 +47,9 @@ object DumpDbUtils { if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) { SQLUtils.executeUpdate(statement, GTFSCommons.T_SERVICE_IDS_SQL_CREATE) } + if (FeatureFlags.F_EXPORT_TRIP_ID_INTS) { + SQLUtils.executeUpdate(statement, GTFSCommons.T_TRIP_IDS_SQL_CREATE) + } SQLUtils.executeUpdate(statement, GTFSCommons.T_SERVICE_DATES_SQL_CREATE) SQLUtils.executeUpdate(statement, GTFSCommons.T_STRINGS_SQL_CREATE) } diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GFieldTypes.kt b/src/main/java/org/mtransit/parser/gtfs/data/GFieldTypes.kt index 46a96e9..cc0868c 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GFieldTypes.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GFieldTypes.kt @@ -11,6 +11,7 @@ import java.util.Locale @Suppress("unused", "MemberVisibilityCanBePrivate") object GFieldTypes { + @Suppress("SpellCheckingInspection") const val TIME_FORMAT_PATTERN = "HHmmss" const val DATE_FORMAT_PATTERN = "yyyyMMdd" diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GRoute.kt b/src/main/java/org/mtransit/parser/gtfs/data/GRoute.kt index 680cec7..59e0d1a 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GRoute.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GRoute.kt @@ -119,6 +119,7 @@ data class GRoute( append(routeShortName) append(")") } + append("}") } fun to() = Route( diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java b/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java index 6b8968b..f3ff751 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java +++ b/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java @@ -342,6 +342,14 @@ public void addTrip(@NotNull GTrip gTrip, @Nullable PreparedStatement insertStop this.tripIdIntsUIDs.put(gTrip.getTripIdInt(), gTrip.getUID()); } + @NotNull + private Collection getAllTripRouteIdInts() { + if (USE_DB_ONLY) { + return GIDs.getInts(GTFSDataBase.selectTripRouteIds()); + } + return this.routeIdIntTripsCache.keySet(); + } + public int readTripsCount() { if (USE_DB_ONLY) { return GTFSDataBase.countTrips(); @@ -805,9 +813,7 @@ public void cleanupExcludedData() { int r = 0; try { final Collection allRouteIdsInt = getAllRouteIdInts(); // this agency & type & not excluded only - final Collection allTripRouteIdInts = - USE_DB_ONLY ? GIDs.getInts(GTFSDataBase.selectTripRouteIds()) - : this.routeIdIntTripsCache.keySet(); + final Collection allTripRouteIdInts = getAllTripRouteIdInts(); for (Integer tripRouteIdInt : allTripRouteIdInts) { if (!allRouteIdsInt.contains(tripRouteIdInt)) { final List routeTrips = getRouteTrips(tripRouteIdInt); @@ -886,19 +892,19 @@ public void cleanupExcludedServiceIds() { int r = 0; try { final Collection allRouteIdsInt = getAllRouteIdInts(); - final Collection allTripIdsInt = getAllRouteIdInts(); - HashSet routeTripServiceIdInts = new HashSet<>(); + final Collection allTripRouteIdInts = getAllTripRouteIdInts(); + final HashSet notExcludedRouteTripServiceIdInts = new HashSet<>(); for (Integer gRouteIdInt : allRouteIdsInt) { - if (allTripIdsInt.contains(gRouteIdInt)) { + if (allTripRouteIdInts.contains(gRouteIdInt)) { for (GTrip gTrip : getRouteTrips(gRouteIdInt)) { - routeTripServiceIdInts.add(gTrip.getServiceIdInt()); + notExcludedRouteTripServiceIdInts.add(gTrip.getServiceIdInt()); } } } final Iterator itGCalendar = getAllCalendars().iterator(); while (itGCalendar.hasNext()) { final GCalendar gCalendar = itGCalendar.next(); - if (!routeTripServiceIdInts.contains(gCalendar.getServiceIdInt())) { + if (!notExcludedRouteTripServiceIdInts.contains(gCalendar.getServiceIdInt())) { itGCalendar.remove(); if (ALL_CALENDARS_STORED_IN_CALENDAR_DATES) { for (GCalendarDate gCalendarDate : gCalendar.flattenToCalendarDates()) { @@ -917,7 +923,7 @@ public void cleanupExcludedServiceIds() { Iterator itGCalendarDate = getAllCalendarDates().iterator(); while (itGCalendarDate.hasNext()) { GCalendarDate gCalendarDate = itGCalendarDate.next(); - if (!routeTripServiceIdInts.contains(gCalendarDate.getServiceIdInt())) { + if (!notExcludedRouteTripServiceIdInts.contains(gCalendarDate.getServiceIdInt())) { itGCalendarDate.remove(); GTFSDataBase.deleteCalendarDate(gCalendarDate.to()); logRemoved("Removed calendar date (or calendar): %s.", gCalendarDate.toStringPlus()); diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GTime.kt b/src/main/java/org/mtransit/parser/gtfs/data/GTime.kt index b429ed1..c598840 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GTime.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GTime.kt @@ -2,7 +2,6 @@ package org.mtransit.parser.gtfs.data import org.mtransit.parser.Constants.EMPTY import org.mtransit.parser.MTLog -import java.lang.Exception import java.util.Calendar import java.util.Date diff --git a/src/main/java/org/mtransit/parser/mt/MGenerator.java b/src/main/java/org/mtransit/parser/mt/MGenerator.java index 9a8f674..af210f4 100644 --- a/src/main/java/org/mtransit/parser/mt/MGenerator.java +++ b/src/main/java/org/mtransit/parser/mt/MGenerator.java @@ -38,6 +38,8 @@ import org.mtransit.parser.mt.data.MDirectionStop; import org.mtransit.parser.mt.data.MString; import org.mtransit.parser.mt.data.MStrings; +import org.mtransit.parser.mt.data.MTripId; +import org.mtransit.parser.mt.data.MTripIds; import java.io.BufferedWriter; import java.io.File; @@ -166,8 +168,8 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age Collections.sort(mStopsList); final ArrayList mRoutesList = new ArrayList<>(mRoutes); Collections.sort(mRoutesList); - final ArrayList mTripsList = new ArrayList<>(mDirections); - Collections.sort(mTripsList); + final ArrayList mDirectionsList = new ArrayList<>(mDirections); + Collections.sort(mDirectionsList); final ArrayList mDirectionStopsList = new ArrayList<>(mDirectionStops); Collections.sort(mDirectionStopsList); final ArrayList mServiceDatesList = new ArrayList<>(mServiceDates); @@ -175,10 +177,11 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age MTLog.log("Generating routes, trips, trip stops & stops objects... DONE"); MTLog.log("- Agencies: %d", mAgenciesList.size()); MTLog.log("- Routes: %d", mRoutesList.size()); - MTLog.log("- Trips: %d", mTripsList.size()); - MTLog.log("- Trip stops: %d", mDirectionStopsList.size()); + MTLog.log("- Directions: %d", mDirectionsList.size()); + MTLog.log("- Direction stops: %d", mDirectionStopsList.size()); MTLog.log("- Stops: %d", mStopsList.size()); MTLog.log("- Service Ids: %d", MServiceIds.count()); + MTLog.log("- Trip Ids: %d", MTripIds.count()); MTLog.log("- Strings: %d", MStrings.count()); MTLog.log("- Service Dates: %d", mServiceDatesList.size()); MTLog.log("- Route with Frequencies: %d", mRouteFrequencies.size()); @@ -188,7 +191,7 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age mAgenciesList, mStopsList, mRoutesList, - mTripsList, + mDirectionsList, mDirectionStopsList, mServiceDatesList, mRouteFrequencies, @@ -208,6 +211,7 @@ private static void logMerging(@NotNull String msg, long routeId) { private static final String GTFS_SCHEDULE = "gtfs_schedule"; private static final String GTFS_SCHEDULE_SERVICE_DATES = GTFS_SCHEDULE + "_service_dates"; // DB private static final String GTFS_SCHEDULE_SERVICE_IDS = GTFS_SCHEDULE + "_service_ids"; // DB + private static final String GTFS_SCHEDULE_TRIP_IDS = GTFS_SCHEDULE + "_trip_ids"; // DB private static final String GTFS_SCHEDULE_STOP = GTFS_SCHEDULE + "_stop_"; // file private static final String GTFS_FREQUENCY = "gtfs_frequency"; private static final String GTFS_FREQUENCY_ROUTE = GTFS_FREQUENCY + "_route_"; // file @@ -284,6 +288,8 @@ public static void dumpFiles(@NotNull GAgencyTools gAgencyTools, dumpScheduleStops(gAgencyTools, mSpec, fileBase, deleteAll, rawDirF); // FREQUENCY ROUTES dumpFrequencyRoutes(gAgencyTools, mSpec, fileBase, deleteAll, rawDirF); + // TRIP IDS + dumpTripIds(mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); // SERVICE IDS dumpServiceIds(mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); // AFTER SCHEDULE STOPS & FREQUENCY ROUTES // STRINGS @@ -523,6 +529,52 @@ private static Pair, Pair> dumpRDSStops(@Nu return minMaxLatLng; } + private static void dumpTripIds( + @Nullable MSpec mSpec, + @NotNull String fileBase, + boolean deleteAll, + @NotNull File dataDirF, + @NotNull File rawDirF, + @Nullable Connection dbConnection) { + if (!FeatureFlags.F_EXPORT_TRIP_ID_INTS) return; + if (!deleteAll + && (mSpec == null || !mSpec.isValid() || (F_PRE_FILLED_DB && dbConnection == null))) { + throw new MTLog.Fatal("Generated data invalid (agencies: %s)!", mSpec); + } + if (F_PRE_FILLED_DB) { + FileUtils.deleteIfExist(new File(rawDirF, fileBase + GTFS_SCHEDULE_TRIP_IDS)); // migration from src/main/res/raw to data + } + final File file = new File(F_PRE_FILLED_DB ? dataDirF : rawDirF, fileBase + GTFS_SCHEDULE_TRIP_IDS); + FileUtils.deleteIfExist(file); // delete previous + if (deleteAll) return; + try (BufferedWriter ow = new BufferedWriter(new FileWriter(file))) { + MTLog.logPOINT(); // LOG + Statement dbStatement = null; + String sqlInsert = null; + if (F_PRE_FILLED_DB) { + SQLUtils.setAutoCommit(dbConnection, false); // START TRANSACTION + dbStatement = dbConnection.createStatement(); + sqlInsert = GTFSCommons.getT_TRIP_IDS_SQL_INSERT(); + } + for (MTripId mTripId : MTripIds.getAll()) { + final String tripIdsInsert = mTripId.toFile(); + if (F_PRE_FILLED_DB) { + SQLUtils.executeUpdate( + dbStatement, + String.format(sqlInsert, tripIdsInsert) + ); + } + ow.write(tripIdsInsert); + ow.write(Constants.NEW_LINE); + } + if (F_PRE_FILLED_DB) { + SQLUtils.setAutoCommit(dbConnection, true); // END TRANSACTION == commit() + } + } catch (Exception ioe) { + throw new MTLog.Fatal(ioe, "I/O Error while writing trip IDs file!"); + } + } + private static void dumpServiceIds( @Nullable MSpec mSpec, @NotNull String fileBase, @@ -576,7 +628,7 @@ private static void dumpStrings( @NotNull File dataDirF, @NotNull File rawDirF, @Nullable Connection dbConnection) { - if (!FeatureFlags.F_EXPORT_STRINGS) return; + if (!FeatureFlags.F_EXPORT_STRINGS && !FeatureFlags.F_EXPORT_SCHEDULE_STRINGS) return; if (!deleteAll && (mSpec == null || !mSpec.isValid() || (F_PRE_FILLED_DB && dbConnection == null))) { throw new MTLog.Fatal("Generated data invalid (agencies: %s)!", mSpec); @@ -743,15 +795,15 @@ private static void dumpScheduleStops(@NotNull GAgencyTools gAgencyTools, } // LOG MSchedule lastSchedule = null; for (MSchedule mSchedule : mStopSchedules) { - if (mSchedule.isSameServiceAndDirection(lastSchedule)) { - ow.write(SQLUtils.COLUMN_SEPARATOR); - ow.write(mSchedule.toFileSameServiceIdAndDirectionId(lastSchedule)); - } else { + if (!mSchedule.isSameServiceAndDirection(lastSchedule)) { + lastSchedule = null; if (!empty) { ow.write(Constants.NEW_LINE); } - ow.write(mSchedule.toFileNewServiceIdAndDirectionId(gAgencyTools)); + } else { + ow.write(SQLUtils.COLUMN_SEPARATOR); } + ow.write(mSchedule.toFile(gAgencyTools, lastSchedule)); empty = false; lastSchedule = mSchedule; } diff --git a/src/main/java/org/mtransit/parser/mt/MReader.kt b/src/main/java/org/mtransit/parser/mt/MReader.kt index 0d9f6fb..36de2a0 100644 --- a/src/main/java/org/mtransit/parser/mt/MReader.kt +++ b/src/main/java/org/mtransit/parser/mt/MReader.kt @@ -15,6 +15,7 @@ import org.mtransit.parser.gtfs.data.GFieldTypes import org.mtransit.parser.mt.data.MServiceDate import org.mtransit.parser.mt.data.MServiceId import org.mtransit.parser.mt.data.MString +import org.mtransit.parser.mt.data.MTripId import java.io.File import java.util.TimeZone @@ -30,14 +31,16 @@ object MReader { private const val RAW = "raw" private const val VALUES = "values" + private const val CURRENT_ = "current_" + private const val NEXT_ = "next_" + private fun getResDirName(fileBase: String? = null): String { - return "$MAIN_SRC_DIR/" + if ("current_".equals(fileBase, ignoreCase = true)) { - "$RES-current" - } else if ("next_".equals(fileBase, ignoreCase = true)) { - "$RES-next" - } else { - RES - } + return "$MAIN_SRC_DIR/" + + when { + CURRENT_.equals(fileBase, ignoreCase = true) -> "$RES-current" + NEXT_.equals(fileBase, ignoreCase = true) -> "$RES-next" + else -> RES + } } // endregion @@ -122,47 +125,43 @@ object MReader { private const val GTFS_SCHEDULE_SERVICE_DATES = "gtfs_schedule_service_dates" @JvmStatic - fun loadServiceDates(fileBase: String): List? { - try { - val gtfsScheduleServiceDates = getResDirName(fileBase) + "/$RAW/${fileBase}$GTFS_SCHEDULE_SERVICE_DATES" - val gtfsScheduleServiceDatesFile = File(gtfsScheduleServiceDates) - if (!gtfsScheduleServiceDatesFile.exists()) { - MTLog.log("File not found '$gtfsScheduleServiceDates'!") - return null - } - val gtfsScheduleServiceDatesFileLines = gtfsScheduleServiceDatesFile.readLines() - val serviceDates = gtfsScheduleServiceDatesFileLines.mapNotNull { line -> - MServiceDate.fromFileLine(line) + fun loadServiceDates(fileBase: String) = + readFile("service dates", fileBase, GTFS_SCHEDULE_SERVICE_DATES) { MServiceDate.fromFileLine(it) } + + private fun readFile(type: String, fileBase: String, fileName: String, transform: (String) -> T?): List? = try { + (File("${getResDirName(fileBase)}/$RAW/${fileBase}$fileName").takeIf { it.exists() } + ?: CURRENT_.takeIf { NEXT_ == fileBase }?.let { File("${getResDirName(it)}/$RAW/${it}$fileName") }?.takeIf { it.exists() } + ?: "".takeIf { CURRENT_ == fileBase || NEXT_ == fileBase }?.let { File("${getResDirName(it)}/$RAW/${it}$fileName") }?.takeIf { it.exists() }) + ?.readLines() + ?.mapNotNull { transform(it) } + ?: run { + MTLog.log("File not found for '$type' with fileBase '$fileBase' and fileName '$fileName'!") + null } - return serviceDates - } catch (e: Exception) { - MTLog.logNonFatal(e, "Error while reading '$fileBase' service dates!") - return null - } + } catch (e: Exception) { + MTLog.logNonFatal(e, "Error while reading '$fileBase' $type!") + null } // endregion + // region trip IDs + + private const val GTFS_SCHEDULE_TRIP_IDS = "gtfs_schedule_trip_ids" + + @JvmStatic + fun loadTripIds(fileBase: String) = + readFile("trip ids", fileBase, GTFS_SCHEDULE_TRIP_IDS) { MTripId.fromFileLine(it) } + + // endregion + // region service IDs private const val GTFS_SCHEDULE_SERVICE_IDS = "gtfs_schedule_service_ids" @JvmStatic - fun loadServiceIds(fileBase: String) = try { - File(getResDirName(fileBase) + "/$RAW/${fileBase}$GTFS_SCHEDULE_SERVICE_IDS") - .takeIf { it.exists() } - ?.readLines() - ?.mapNotNull { line -> - MServiceId.fromFileLine(line) - } - ?: run { - MTLog.log("File not found '${"/$RAW/${fileBase}$GTFS_SCHEDULE_SERVICE_IDS"}'!") - null - } - } catch (e: Exception) { - MTLog.logNonFatal(e, "Error while reading '$fileBase' service ids!") - null - } + fun loadServiceIds(fileBase: String) = + readFile("service ids", fileBase, GTFS_SCHEDULE_SERVICE_IDS) { MServiceId.fromFileLine(it) } // endregion @@ -171,21 +170,8 @@ object MReader { private const val GTFS_STRINGS = "gtfs_strings" @JvmStatic - fun loadStrings(fileBase: String) = try { - File(getResDirName(fileBase) + "/$RAW/${fileBase}$GTFS_STRINGS") - .takeIf { it.exists() } - ?.readLines() - ?.mapNotNull { line -> - MString.fromFileLine(line) - } - ?: run { - MTLog.log("File not found '${"/$RAW/${fileBase}$GTFS_STRINGS"}'!") - null - } - } catch (e: Exception) { - MTLog.logNonFatal(e, "Error while reading '$fileBase' strings!") - null - } + fun loadStrings(fileBase: String) = + readFile("strings", fileBase, GTFS_STRINGS) { MString.fromFileLine(it) } // endregion } \ No newline at end of file diff --git a/src/main/java/org/mtransit/parser/mt/data/MDirection.kt b/src/main/java/org/mtransit/parser/mt/data/MDirection.kt index be6f4e0..9e2a3b0 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MDirection.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MDirection.kt @@ -1,5 +1,6 @@ package org.mtransit.parser.mt.data +import org.mtransit.commons.FeatureFlags import org.mtransit.commons.sql.SQLUtils import org.mtransit.parser.MTLog import org.mtransit.parser.db.SQLUtils.quotesEscape @@ -177,7 +178,7 @@ data class MDirection( fun toFile() = listOf( id.toString(), // ID headsignType.toString(), // HEADSIGN TYPE - headsignValue.toStringIds().quotesEscape(), // HEADSIGN STRING + headsignValue.toStringIds(FeatureFlags.F_EXPORT_STRINGS).quotesEscape(), // HEADSIGN STRING routeId.toString(), // ROUTE ID ).joinToString(SQLUtils.COLUMN_SEPARATOR) diff --git a/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt b/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt index 01d4637..b48993e 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt @@ -1,9 +1,7 @@ package org.mtransit.parser.mt.data import androidx.annotation.Discouraged -import org.mtransit.commons.FeatureFlags import org.mtransit.commons.sql.SQLUtils -import org.mtransit.parser.db.SQLUtils.quotesEscape import org.mtransit.parser.gtfs.GAgencyTools import org.mtransit.parser.gtfs.data.GIDs @@ -25,11 +23,7 @@ data class MFrequency( val uID by lazy { getNewUID(serviceIdInt, directionId, startTime, endTime) } fun toFile(agencyTools: GAgencyTools) = buildList { - if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) { - add(MServiceIds.getInt(agencyTools.cleanServiceId(_serviceId))) - } else { - add(agencyTools.cleanServiceId(_serviceId).quotesEscape()) - } + add(MServiceIds.convert(agencyTools.cleanServiceId(_serviceId))) add(directionId.toString()) add(startTime.toString()) add(endTime.toString()) diff --git a/src/main/java/org/mtransit/parser/mt/data/MRoute.kt b/src/main/java/org/mtransit/parser/mt/data/MRoute.kt index 00bdc7b..2f7c276 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MRoute.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MRoute.kt @@ -1,5 +1,6 @@ package org.mtransit.parser.mt.data +import org.mtransit.commons.FeatureFlags import org.mtransit.commons.GTFSCommons import org.mtransit.commons.sql.SQLUtils import org.mtransit.parser.db.SQLUtils.quotes @@ -37,8 +38,8 @@ data class MRoute( fun toFile() = buildList { add(id.toString()) // ID - add(shortName.orEmpty().toStringIds().quotesEscape()) // short name - add(longName.toStringIds().quotesEscape()) // long name + add(shortName.orEmpty().toStringIds(FeatureFlags.F_EXPORT_STRINGS).quotesEscape()) // short name + add(longName.toStringIds(FeatureFlags.F_EXPORT_STRINGS).quotesEscape()) // long name add((color?.uppercase().orEmpty()).quotes()) // color add(originalIdHash.toString()) // original ID hash add(type.toString()) diff --git a/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt b/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt index 85ad743..e3704e1 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt @@ -3,7 +3,6 @@ package org.mtransit.parser.mt.data import androidx.annotation.Discouraged import org.mtransit.commons.FeatureFlags import org.mtransit.commons.sql.SQLUtils -import org.mtransit.parser.DefaultAgencyTools import org.mtransit.parser.MTLog import org.mtransit.parser.Pair import org.mtransit.parser.db.SQLUtils.quotes @@ -16,16 +15,14 @@ data class MSchedule( val serviceIdInt: Int, val directionId: Long, val stopId: Int, - val arrival: Int, - val departure: Int, + val arrival: Int, // HHmmss + val departure: Int, // HHmmss val tripIdInt: Int, val accessible: Int, var headsignType: Int = -1, var headsignValue: String? = null, ) : Comparable { - private val arrivalBeforeDeparture: Int = departure - arrival - constructor( routeId: Long, serviceIdInt: Int, @@ -97,46 +94,44 @@ data class MSchedule( "+(uID:$uID)" } - fun toFileNewServiceIdAndDirectionId(agencyTools: GAgencyTools) = buildList { - add( - if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) { - MServiceIds.getInt(agencyTools.cleanServiceId(_serviceId)) - } else { - agencyTools.cleanServiceId(_serviceId).quotesEscape() - } - ) - // no route ID, just for file split - add(directionId.toString()) - add(departure.toString()) - if (DefaultAgencyTools.EXPORT_TRIP_ID) { - @Suppress("ControlFlowWithEmptyBody") - if (arrivalBeforeDeparture > 0) { - // TODO ? - } - add(arrivalBeforeDeparture.takeIf { it > 0 }?.toString().orEmpty()) // arrival before departure - add(_tripId.quotesEscape()) + fun toFile(agencyTools: GAgencyTools, lastSchedule: MSchedule?) = buildList { + if (lastSchedule == null) { // NEW + add(MServiceIds.convert(agencyTools.cleanServiceId(_serviceId))) + // no route ID, just for file split + add(directionId.toString()) } - add(headsignType.takeIf { it >= 0 }?.toString().orEmpty()) - add(headsignValue.orEmpty().toStringIds().quotesEscape()) - add(accessible.toString()) - }.joinToString(SQLUtils.COLUMN_SEPARATOR) - - fun toFileSameServiceIdAndDirectionId(lastSchedule: MSchedule?) = buildList { - add((departure - (lastSchedule?.departure ?: 0)).toString()) - if (DefaultAgencyTools.EXPORT_TRIP_ID) { - @Suppress("ControlFlowWithEmptyBody") - if (arrivalBeforeDeparture > 0) { - // TODO ? + val lastDeparture = if (FeatureFlags.F_SCHEDULE_IN_MINUTES) { + lastSchedule?.departure?.div(100)?.times(100) + } else { + lastSchedule?.departure + } ?: 0 + if (FeatureFlags.F_SCHEDULE_IN_MINUTES) { + add((departure - lastDeparture).div(100).toString()) // truncates the time to an minute that is closer to 0 + } else { + add((departure - lastDeparture).toString()) + } + if (FeatureFlags.F_EXPORT_TRIP_ID) { + var arrivalDiff = (departure - arrival).takeIf { it > MIN_ARRIVAL_DIFF_IN_HH_MM_SS } + if (FeatureFlags.F_SCHEDULE_IN_MINUTES) { + arrivalDiff = arrivalDiff?.div(100) // truncates the time to an minute that is closer to 0 } - add(arrivalBeforeDeparture.takeIf { it > 0 }?.toString().orEmpty()) // arrival before departure - add(_tripId.quotesEscape()) + add(arrivalDiff?.toString().orEmpty()) + add(MTripIds.convert(_tripId)) } if (headsignType == MDirection.HEADSIGN_TYPE_NO_PICKUP) { add(MDirection.HEADSIGN_TYPE_NO_PICKUP.toString()) - add(MDirection.HEADSIGN_DEFAULT_VALUE.quotes()) + if (FeatureFlags.F_SCHEDULE_NO_QUOTES) { + add(MDirection.HEADSIGN_DEFAULT_VALUE) + } else { + add(MDirection.HEADSIGN_DEFAULT_VALUE.quotes()) + } } else { add(headsignType.takeIf { it >= 0 }?.toString().orEmpty()) - add(headsignValue.orEmpty().toStringIds().quotesEscape()) + if (FeatureFlags.F_SCHEDULE_NO_QUOTES) { + add(headsignValue.orEmpty().toStringIds(FeatureFlags.F_EXPORT_STRINGS || FeatureFlags.F_EXPORT_SCHEDULE_STRINGS)) + } else { + add(headsignValue.orEmpty().toStringIds(FeatureFlags.F_EXPORT_STRINGS || FeatureFlags.F_EXPORT_SCHEDULE_STRINGS).quotesEscape()) + } } add(accessible.toString()) }.joinToString(SQLUtils.COLUMN_SEPARATOR) @@ -196,5 +191,7 @@ data class MSchedule( stopId: Int, departure: Int ) = "${serviceIdInt}$UID_SEPARATOR${directionId}$UID_SEPARATOR${stopId}$UID_SEPARATOR${departure}" + + const val MIN_ARRIVAL_DIFF_IN_HH_MM_SS = 100 // 1 minute } } diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt index 0ca1090..3d9883e 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt @@ -1,9 +1,7 @@ package org.mtransit.parser.mt.data import androidx.annotation.Discouraged -import org.mtransit.commons.FeatureFlags import org.mtransit.commons.sql.SQLUtils -import org.mtransit.parser.db.SQLUtils.quotesEscape import org.mtransit.parser.db.SQLUtils.unquotes import org.mtransit.parser.gtfs.GAgencyTools import org.mtransit.parser.gtfs.data.GCalendarDate @@ -40,11 +38,7 @@ data class MServiceDate( ).compare(this, other) fun toFile(agencyTools: GAgencyTools) = buildList { - if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) { - add(MServiceIds.getInt(agencyTools.cleanServiceId(_serviceId))) - } else { - add(agencyTools.cleanServiceId(_serviceId).quotesEscape()) - } + add(MServiceIds.convert(agencyTools.cleanServiceId(_serviceId))) add(calendarDate.toString()) add(exceptionType.toString()) }.joinToString(SQLUtils.COLUMN_SEPARATOR) diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt index 5692384..a96149e 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt @@ -2,7 +2,9 @@ package org.mtransit.parser.mt.data import androidx.collection.SparseArrayCompat import androidx.collection.mutableScatterMapOf +import org.mtransit.commons.FeatureFlags import org.mtransit.parser.MTLog +import org.mtransit.parser.db.SQLUtils.quotesEscape object MServiceIds { @@ -36,7 +38,7 @@ object MServiceIds { @Suppress("unused") @JvmStatic - fun getString(serviceIdInt: Int) = + fun getId(serviceIdInt: Int) = idIntToId[serviceIdInt] ?: throw MTLog.Fatal("Unexpected Service ID integer $serviceIdInt!") @JvmStatic @@ -55,4 +57,12 @@ object MServiceIds { add(MServiceId(idInt, id)) } }.sorted() + + @JvmStatic + fun convert(serviceId: String) = + if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) { + getInt(serviceId).toString() + } else { + serviceId.quotesEscape() + } } diff --git a/src/main/java/org/mtransit/parser/mt/data/MStop.kt b/src/main/java/org/mtransit/parser/mt/data/MStop.kt index 0dae53b..5ed630e 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MStop.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MStop.kt @@ -1,5 +1,6 @@ package org.mtransit.parser.mt.data +import org.mtransit.commons.FeatureFlags import org.mtransit.commons.GTFSCommons import org.mtransit.commons.sql.SQLUtils import org.mtransit.parser.db.SQLUtils.quotesEscape @@ -35,18 +36,14 @@ data class MStop( GTFSCommons.stringIdToHash(originalId), ) - fun hasLat(): Boolean { - return lat != 0.0 - } + fun hasLat() = lat != 0.0 - fun hasLng(): Boolean { - return lng != 0.0 - } + fun hasLng() = lng != 0.0 fun toFile() = listOf( id.toString(), // ID code.quotesEscape(), // code - name.toStringIds().quotesEscape(), // name + name.toStringIds(FeatureFlags.F_EXPORT_STRINGS).quotesEscape(), // name MDataChangedManager.avoidLatLngChanged(lat), // latitude MDataChangedManager.avoidLatLngChanged(lng), // longitude accessible.toString(), diff --git a/src/main/java/org/mtransit/parser/mt/data/MStrings.kt b/src/main/java/org/mtransit/parser/mt/data/MStrings.kt index 394f690..5c2ea92 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MStrings.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MStrings.kt @@ -2,7 +2,6 @@ package org.mtransit.parser.mt.data import androidx.collection.SparseArrayCompat import androidx.collection.mutableScatterMapOf -import org.mtransit.commons.FeatureFlags import org.mtransit.commons.GTFSCommons import org.mtransit.parser.MTLog @@ -65,8 +64,8 @@ object MStrings { }.sorted() @JvmStatic - fun convert(strings: String): String { - if (!FeatureFlags.F_EXPORT_STRINGS) return strings + fun convert(strings: String, enabled: Boolean): String { + if (!enabled) return strings if (strings.isEmpty()) return strings return strings .split(GTFSCommons.STRINGS_SEPARATOR) @@ -75,4 +74,4 @@ object MStrings { } } -fun String.toStringIds() = MStrings.convert(this) \ No newline at end of file +fun String.toStringIds(enabled: Boolean) = MStrings.convert(this, enabled) \ No newline at end of file diff --git a/src/main/java/org/mtransit/parser/mt/data/MTripId.kt b/src/main/java/org/mtransit/parser/mt/data/MTripId.kt new file mode 100644 index 0000000..173dfd1 --- /dev/null +++ b/src/main/java/org/mtransit/parser/mt/data/MTripId.kt @@ -0,0 +1,41 @@ +package org.mtransit.parser.mt.data + +import org.mtransit.commons.sql.SQLUtils +import org.mtransit.commons.sql.SQLUtils.unquotesUnescape +import org.mtransit.parser.MTLog +import org.mtransit.parser.db.SQLUtils.quotesEscape + +data class MTripId( + val tripIdInt: Int, + val tripId: String, // already agencyTools.cleanTripId(tripId) before +) : Comparable { + + /** + * same order as [org.mtransit.commons.GTFSCommons.T_TRIP_IDS_SQL_INSERT] + */ + fun toFile() = buildList { + add(tripIdInt.toString()) + add(tripId.quotesEscape()) // already agencyTools.cleanTripId(tripId) before + }.joinToString(SQLUtils.COLUMN_SEPARATOR) + + override fun compareTo(other: MTripId) = compareBy( + MTripId::tripIdInt, + MTripId::tripId, + ).compare(this, other) + + companion object { + fun fromFileLine(line: String) = + line.split(SQLUtils.COLUMN_SEPARATOR) + .takeIf { it.size == 2 } + ?.let { columns -> + MTripId( + tripIdInt = columns[0].toInt(), + tripId = columns[1].unquotesUnescape(), + ) + } + ?: run { + MTLog.log("Invalid trip ID line: '$line'!") + null + } + } +} diff --git a/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt b/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt new file mode 100644 index 0000000..2b492f1 --- /dev/null +++ b/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt @@ -0,0 +1,68 @@ +package org.mtransit.parser.mt.data + +import androidx.collection.SparseArrayCompat +import androidx.collection.mutableScatterMapOf +import org.mtransit.commons.FeatureFlags +import org.mtransit.parser.MTLog +import org.mtransit.parser.db.SQLUtils.quotesEscape + +object MTripIds { + + private val incrementLock = Any() + + private var increment = 0 + private val idIntToId = SparseArrayCompat() + private val idToIdInt = mutableScatterMapOf() + + @JvmStatic + fun addAll(lastTripIds: List?) { + lastTripIds?.forEach { add(it) } + } + + fun add(tripId: MTripId) { + synchronized(incrementLock) { + idIntToId.put(tripId.tripIdInt, tripId.tripId) + idToIdInt[tripId.tripId] = tripId.tripIdInt + increment = maxOf(increment, tripId.tripIdInt) + } + } + + fun add(tripId: String): Int { + synchronized(incrementLock) { + increment++ // move to next + val newTripId = MTripId(increment, tripId) + add(newTripId) + return newTripId.tripIdInt + } + } + + @Suppress("unused") + @JvmStatic + fun getId(tripIdInt: Int) = + idIntToId[tripIdInt] ?: throw MTLog.Fatal("Unexpected Trip ID integer $tripIdInt!") + + @JvmStatic + fun getInt(tripId: String): Int = + idToIdInt[tripId] + ?: synchronized(incrementLock) { + return idToIdInt[tripId] ?: add(tripId) + } + + @JvmStatic + fun count() = idIntToId.size() + + @JvmStatic + fun getAll() = buildList { + idToIdInt.forEach { id, idInt -> + add(MTripId(idInt, id)) + } + }.sorted() + + @JvmStatic + fun convert(tripId: String) = + if (FeatureFlags.F_EXPORT_TRIP_ID_INTS) { + getInt(tripId).toString() + } else { + tripId.quotesEscape() + } +}