From f9c1d136e9eaed940fb5967fe6f86e6000172aec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Thu, 4 Dec 2025 15:22:17 -0500 Subject: [PATCH 1/4] Vehicles locations --- .../java/org/mtransit/parser/db/DBUtils.kt | 114 +++-------- .../parser/mt/GenerateMObjectsTask.java | 14 +- .../mtransit/parser/mt/MDataChangedManager.kt | 2 +- .../org/mtransit/parser/mt/MGenerator.java | 177 +++++++++++++----- .../org/mtransit/parser/mt/data/MSchedule.kt | 43 +++-- .../mtransit/parser/mt/data/MServiceDate.kt | 37 +++- .../java/org/mtransit/parser/mt/data/MSpec.kt | 1 + .../java/org/mtransit/parser/mt/data/MTrip.kt | 67 +++++++ 8 files changed, 292 insertions(+), 163 deletions(-) create mode 100644 src/main/java/org/mtransit/parser/mt/data/MTrip.kt diff --git a/src/main/java/org/mtransit/parser/db/DBUtils.kt b/src/main/java/org/mtransit/parser/db/DBUtils.kt index 77249c0..de1d893 100644 --- a/src/main/java/org/mtransit/parser/db/DBUtils.kt +++ b/src/main/java/org/mtransit/parser/db/DBUtils.kt @@ -151,17 +151,17 @@ object DBUtils { @JvmStatic fun insertStopTime(gStopTime: GStopTime, preparedStatement: PreparedStatement) { try { - var idx = 1 + var idx = 0 with(preparedStatement) { - setInt(idx++, gStopTime.tripIdInt) - setInt(idx++, gStopTime.stopIdInt) - setInt(idx++, gStopTime.stopSequence) - setInt(idx++, gStopTime.arrivalTime) - setInt(idx++, gStopTime.departureTime) - setString(idx++, gStopTime.stopHeadsign?.quotesEscape()) - setInt(idx++, gStopTime.pickupType.id) - setInt(idx++, gStopTime.dropOffType.id) - setInt(idx++, gStopTime.timePoint.id) + setInt(++idx, gStopTime.tripIdInt) + setInt(++idx, gStopTime.stopIdInt) + setInt(++idx, gStopTime.stopSequence) + setInt(++idx, gStopTime.arrivalTime) + setInt(++idx, gStopTime.departureTime) + setString(++idx, gStopTime.stopHeadsign?.quotesEscape()) + setInt(++idx, gStopTime.pickupType.id) + setInt(++idx, gStopTime.dropOffType.id) + setInt(++idx, gStopTime.timePoint.id) addBatch() } insertRowCount++ @@ -434,21 +434,11 @@ object DBUtils { // SERVICE ID serviceIdInt?.let { @Suppress("KotlinConstantConditions") - query += if (whereAdded) { - " AND" - } else { - " WHERE" - } - whereAdded = true + query += if (whereAdded) " AND" else " WHERE"; whereAdded = true query += " ${MSchedule.SERVICE_ID} = $serviceIdInt" } serviceIdInts?.let { - query += if (whereAdded) { - " AND" - } else { - " WHERE" - } - whereAdded = true + query += if (whereAdded) " AND" else " WHERE"; whereAdded = true query += " ${MSchedule.SERVICE_ID} IN ${ serviceIdInts .distinct() @@ -458,26 +448,15 @@ object DBUtils { postfix = ")" ) { "$it" } }" - whereAdded = true } // DIRECTION ID directionId?.let { - query += if (whereAdded) { - " AND" - } else { - " WHERE" - } - whereAdded = true + query += if (whereAdded) " AND" else " WHERE"; whereAdded = true query += " ${MSchedule.DIRECTION_ID} = $directionId" } directionIds?.let { - query += if (whereAdded) { - " AND" - } else { - " WHERE" - } - whereAdded = true + query += if (whereAdded) " AND" else " WHERE"; whereAdded = true query += " ${MSchedule.DIRECTION_ID} IN ${ directionIds .distinct() @@ -487,26 +466,15 @@ object DBUtils { postfix = ")" ) { "$it" } }" - whereAdded = true } // STOP ID stopIdInt?.let { - query += if (whereAdded) { - " AND" - } else { - " WHERE" - } - whereAdded = true + query += if (whereAdded) " AND" else " WHERE"; whereAdded = true query += " ${MSchedule.STOP_ID} = $stopIdInt" } stopIdInts?.let { - query += if (whereAdded) { - " AND" - } else { - " WHERE" - } - whereAdded = true + query += if (whereAdded) " AND" else " WHERE"; whereAdded = true query += " ${MSchedule.STOP_ID} IN ${ stopIdInts .distinct() @@ -516,25 +484,14 @@ object DBUtils { postfix = ")" ) { "$it" } }" - whereAdded = true } // ARRIVAL & DEPARTURE arrival?.let { - query += if (whereAdded) { - " AND" - } else { - " WHERE" - } - whereAdded = true + query += if (whereAdded) " AND" else " WHERE"; whereAdded = true query += " ${MSchedule.ARRIVAL} = $arrival" } departure?.let { - query += if (whereAdded) { - " AND" - } else { - " WHERE" - } - whereAdded = true + query += if (whereAdded) " AND" else " WHERE" query += " ${MSchedule.DEPARTURE} = $departure" } query += " ORDER BY " + @@ -587,52 +544,27 @@ object DBUtils { var whereAdded = false serviceIdInt?.let { @Suppress("KotlinConstantConditions") - query += if (whereAdded) { - " AND" - } else { - " WHERE" - } - whereAdded = true + query += if (whereAdded) " AND" else " WHERE"; whereAdded = true query += " ${MSchedule.SERVICE_ID} = $serviceIdInt" } directionId?.let { - query += if (whereAdded) { - " AND" - } else { - " WHERE" - } - whereAdded = true + query += if (whereAdded) " AND" else " WHERE"; whereAdded = true query += " ${MSchedule.DIRECTION_ID} = $directionId" } // STOP ID stopIdInt?.let { - query += if (whereAdded) { - " AND" - } else { - " WHERE" - } - whereAdded = true + query += if (whereAdded) " AND" else " WHERE"; whereAdded = true query += " ${MSchedule.STOP_ID} = $stopIdInt" } // ARRIVAL & DEPARTURE arrival?.let { - query += if (whereAdded) { - " AND" - } else { - " WHERE" - } - whereAdded = true + query += if (whereAdded) " AND" else " WHERE"; whereAdded = true query += " ${MSchedule.ARRIVAL} = $arrival" } departure?.let { - query += if (whereAdded) { - " AND" - } else { - " WHERE" - } - whereAdded = true + query += if (whereAdded) " AND" else " WHERE" query += " ${MSchedule.DEPARTURE} = $departure" } deleteCount++ diff --git a/src/main/java/org/mtransit/parser/mt/GenerateMObjectsTask.java b/src/main/java/org/mtransit/parser/mt/GenerateMObjectsTask.java index 7095f65..a810a71 100644 --- a/src/main/java/org/mtransit/parser/mt/GenerateMObjectsTask.java +++ b/src/main/java/org/mtransit/parser/mt/GenerateMObjectsTask.java @@ -33,6 +33,7 @@ import org.mtransit.parser.mt.data.MServiceDate; import org.mtransit.parser.mt.data.MSpec; import org.mtransit.parser.mt.data.MStop; +import org.mtransit.parser.mt.data.MTrip; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -92,6 +93,7 @@ private MSpec doCall() { HashMap mRoutes = new HashMap<>(); HashMap mDirections = new HashMap<>(); HashMap allMDirectionStops = new HashMap<>(); + HashMap mTrips = new HashMap<>(); HashMap mStops = new HashMap<>(); HashSet directionStopIds = new HashSet<>(); // the list of stop IDs used by directions HashSet serviceIdInts = new HashSet<>(); @@ -136,6 +138,7 @@ private MSpec doCall() { mAgencies, mRoutes, mDirections, + mTrips, mStops, allMDirectionStops, directionStopIds, @@ -168,6 +171,8 @@ private MSpec doCall() { Collections.sort(mAgenciesList); ArrayList mStopsList = new ArrayList<>(mStops.values()); Collections.sort(mStopsList); + ArrayList mTripsList = new ArrayList<>(mTrips.values()); + Collections.sort(mTripsList); ArrayList mRoutesList = new ArrayList<>(mRoutes.values()); Collections.sort(mRoutesList); ArrayList mDirectionsList = new ArrayList<>(mDirections.values()); @@ -176,7 +181,7 @@ private MSpec doCall() { Collections.sort(mDirectionStopsList); setDirectionStopNoPickup(mDirectionStopsList, mSchedules.values()); ArrayList mServiceDatesList = new ArrayList<>(mServiceDates); - Collections.sort(mServiceDatesList); + mServiceDatesList.sort(MServiceDate.getCOMPARATOR_BY_CALENDAR_DATE()); ArrayList mFrequenciesList = new ArrayList<>(mFrequencies.values()); Collections.sort(mFrequenciesList); TreeMap> mRouteFrequencies = new TreeMap<>(); @@ -257,12 +262,13 @@ private MSpec doCall() { throw new MTLog.Fatal(e, "Error while parsing dates '%s %s'!", lastCalendarDate, lastDeparture); } } - MSpec mRouteSpec = new MSpec( + final MSpec mRouteSpec = new MSpec( mAgenciesList, mStopsList, mRoutesList, mDirectionsList, mDirectionStopsList, + mTripsList, mServiceDatesList, mRouteFrequencies, firstTimestamp, @@ -279,6 +285,7 @@ private void parseRDS(HashMap mSchedules, HashMap mAgencies, HashMap mRoutes, HashMap mDirections, + HashMap mTrips, HashMap mStops, HashMap allMDirectionStops, HashSet directionStopIds, @@ -341,6 +348,7 @@ private void parseRDS(HashMap mSchedules, mSchedules, mFrequencies, mDirections, + mTrips, mStops, serviceIdInts, mRoute, @@ -450,6 +458,7 @@ private void fixRouteLongName(HashMap mRoutes, HashMap mSchedules, HashMap mFrequencies, HashMap mDirections, + HashMap mTrips, HashMap mStops, HashSet serviceIdInts, MRoute mRoute, @@ -603,6 +612,7 @@ private void parseGTrips(HashMap mSchedules, continue; } mDirections.put(mDirection.getId(), mDirection); + mTrips.put(gTrip.getTripIdInt(), MTrip.from(routeId, mDirection.getId(), gTrip)); } if (g++ % 10 == 0) { // LOG MTLog.logPOINT(); // LOG diff --git a/src/main/java/org/mtransit/parser/mt/MDataChangedManager.kt b/src/main/java/org/mtransit/parser/mt/MDataChangedManager.kt index 540d2b5..c7ba32d 100644 --- a/src/main/java/org/mtransit/parser/mt/MDataChangedManager.kt +++ b/src/main/java/org/mtransit/parser/mt/MDataChangedManager.kt @@ -188,7 +188,7 @@ object MDataChangedManager { } @Suppress("LocalVariableName") val DATE_FORMAT = GFieldTypes.makeDateFormat() - removedCalendarsServiceDates.sortedDescending().forEach { removedServiceDate -> + removedCalendarsServiceDates.sortedWith(MServiceDate.COMPARATOR_BY_CALENDAR_DATE).reversed().forEach { removedServiceDate -> if (ALL_CALENDARS_IN_CALENDAR_DATES) { return } diff --git a/src/main/java/org/mtransit/parser/mt/MGenerator.java b/src/main/java/org/mtransit/parser/mt/MGenerator.java index af210f4..cbd5dbb 100644 --- a/src/main/java/org/mtransit/parser/mt/MGenerator.java +++ b/src/main/java/org/mtransit/parser/mt/MGenerator.java @@ -38,6 +38,7 @@ import org.mtransit.parser.mt.data.MDirectionStop; import org.mtransit.parser.mt.data.MString; import org.mtransit.parser.mt.data.MStrings; +import org.mtransit.parser.mt.data.MTrip; import org.mtransit.parser.mt.data.MTripId; import org.mtransit.parser.mt.data.MTripIds; @@ -80,6 +81,7 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age HashSet mRoutes = new HashSet<>(); // use set to avoid duplicates HashSet mDirections = new HashSet<>(); // use set to avoid duplicates HashSet mDirectionStops = new HashSet<>(); // use set to avoid duplicates + HashSet mTrips = new HashSet<>(); // use set to avoid duplicates HashMap mStops = new HashMap<>(); TreeMap> mRouteFrequencies = new TreeMap<>(); HashSet mServiceDates = new HashSet<>(); // use set to avoid duplicates @@ -106,6 +108,7 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age mRoutes.addAll(mRouteSpec.getRoutes()); mDirections.addAll(mRouteSpec.getDirections()); mDirectionStops.addAll(mRouteSpec.getDirectionStops()); + mTrips.addAll(mRouteSpec.getTrips()); logMerging("stops...", mRouteId); for (MStop mStop : mRouteSpec.getStops()) { if (mStops.containsKey(mStop.getId())) { @@ -168,17 +171,20 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age Collections.sort(mStopsList); final ArrayList mRoutesList = new ArrayList<>(mRoutes); Collections.sort(mRoutesList); + final ArrayList mTripsList = new ArrayList<>(mTrips); + Collections.sort(mTripsList); final ArrayList mDirectionsList = new ArrayList<>(mDirections); Collections.sort(mDirectionsList); final ArrayList mDirectionStopsList = new ArrayList<>(mDirectionStops); Collections.sort(mDirectionStopsList); final ArrayList mServiceDatesList = new ArrayList<>(mServiceDates); - Collections.sort(mServiceDatesList); + mServiceDatesList.sort(MServiceDate.getCOMPARATOR_FOR_FILE()); MTLog.log("Generating routes, trips, trip stops & stops objects... DONE"); MTLog.log("- Agencies: %d", mAgenciesList.size()); MTLog.log("- Routes: %d", mRoutesList.size()); MTLog.log("- Directions: %d", mDirectionsList.size()); MTLog.log("- Direction stops: %d", mDirectionStopsList.size()); + MTLog.log("- Trips: %d", mTripsList.size()); MTLog.log("- Stops: %d", mStopsList.size()); MTLog.log("- Service Ids: %d", MServiceIds.count()); MTLog.log("- Trip Ids: %d", MTripIds.count()); @@ -193,6 +199,7 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age mRoutesList, mDirectionsList, mDirectionStopsList, + mTripsList, mServiceDatesList, mRouteFrequencies, firstTimestamp, @@ -211,7 +218,7 @@ private static void logMerging(@NotNull String msg, long routeId) { private static final String GTFS_SCHEDULE = "gtfs_schedule"; private static final String GTFS_SCHEDULE_SERVICE_DATES = GTFS_SCHEDULE + "_service_dates"; // DB private static final String GTFS_SCHEDULE_SERVICE_IDS = GTFS_SCHEDULE + "_service_ids"; // DB - private static final String GTFS_SCHEDULE_TRIP_IDS = GTFS_SCHEDULE + "_trip_ids"; // DB + private static final String GTFS_SCHEDULE_TRIP_IDS = GTFS_SCHEDULE + "_path_ids"; // do not change to avoid breaking compat w/ old modules // DB private static final String GTFS_SCHEDULE_STOP = GTFS_SCHEDULE + "_stop_"; // file private static final String GTFS_FREQUENCY = "gtfs_frequency"; private static final String GTFS_FREQUENCY_ROUTE = GTFS_FREQUENCY + "_route_"; // file @@ -219,6 +226,7 @@ private static void logMerging(@NotNull String msg, long routeId) { private static final String GTFS_RDS_ROUTES = GTFS_RDS + "_routes"; // DB private static final String GTFS_RDS_DIRECTIONS = GTFS_RDS + "_trips"; // do not change to avoid breaking compat w/ old modules // DB private static final String GTFS_RDS_DIRECTION_STOPS = GTFS_RDS + "_trip_stops"; // do not change to avoid breaking compat w/ old modules // DB + private static final String GTFS_RDS_TRIPS = GTFS_RDS + "_paths"; // do not change to avoid breaking compat w/ old modules // DB private static final String GTFS_RDS_STOPS = GTFS_RDS + "_stops"; // DB private static final int FILE_WRITER_LOG = 10; @@ -278,12 +286,12 @@ public static void dumpFiles(@NotNull GAgencyTools gAgencyTools, dumpRDSDirections(mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); // DIRECTION STOPS dumpRDSDirectionStops(mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); + // TRIPS + dumpRDSTrips(gAgencyTools, mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); // after ROUTE & DIRECTION (FK) // STOPS - Pair, Pair> minMaxLatLng = - dumpRDSStops(mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); + Pair, Pair> minMaxLatLng = dumpRDSStops(mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); // SCHEDULE SERVICE DATES - Pair minMaxDates = - dumpScheduleServiceDates(gAgencyTools, mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); + final Pair minMaxDates = dumpScheduleServiceDates(gAgencyTools, mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); // SCHEDULE STOPS dumpScheduleStops(gAgencyTools, mSpec, fileBase, deleteAll, rawDirF); // FREQUENCY ROUTES @@ -460,6 +468,73 @@ private static void dumpRDSDirectionStops(@Nullable MSpec mSpec, } } + private static void dumpRDSTrips( + @NotNull GAgencyTools gAgencyTools, + @Nullable MSpec mSpec, + @NotNull String fileBase, + boolean deleteAll, + @NotNull File dataDirF, + @NotNull File rawDirF, + @Nullable Connection dbConnection + ) { + if (!FeatureFlags.F_EXPORT_TRIP_ID) return; + if (!deleteAll + && (mSpec == null || !mSpec.isValid() || (F_PRE_FILLED_DB && dbConnection == null))) { + throw new MTLog.Fatal("Generated data invalid (agencies: %s)!", mSpec); + } + File file; + boolean empty; + if (F_PRE_FILLED_DB) { + FileUtils.deleteIfExist(new File(rawDirF, fileBase + GTFS_RDS_TRIPS)); // migration from src/main/res/raw to data + } + file = new File(F_PRE_FILLED_DB ? dataDirF : rawDirF, fileBase + GTFS_RDS_TRIPS); + FileUtils.deleteIfExist(file); // delete previous + if (deleteAll) return; + try (BufferedWriter ow = new BufferedWriter(new FileWriter(file))) { + empty = true; + MTLog.logPOINT(); // LOG + Statement dbStatement = null; + String sqlInsert = null; + if (F_PRE_FILLED_DB) { + SQLUtils.setAutoCommit(dbConnection, false); // START TRANSACTION + dbStatement = dbConnection.createStatement(); + sqlInsert = GTFSCommons.getT_TRIP_SQL_INSERT(); + } + MTrip lastTrip = null; + for (MTrip mTrip : mSpec.getTrips()) { + if (!mTrip.isSameRouteDirectionService(lastTrip)) { + lastTrip = null; + if (!empty) { + ow.write(Constants.NEW_LINE); + } + } else { + ow.write(SQLUtils.COLUMN_SEPARATOR); + } + final String tripInsert = mTrip.toFile(gAgencyTools, lastTrip); + if (F_PRE_FILLED_DB) { + SQLUtils.executeUpdate( + dbStatement, + String.format(sqlInsert, tripInsert) + ); + } + ow.write(tripInsert); + empty = false; + // ow.write(Constants.NEW_LINE); + lastTrip = mTrip; + } + if (empty) { + FileUtils.delete(file); + } else { + ow.write(Constants.NEW_LINE); // GIT convention for easier diff (ELSE unchanged line might appear as changed when not) + } + if (F_PRE_FILLED_DB) { + SQLUtils.setAutoCommit(dbConnection, true); // END TRANSACTION == commit() + } + } catch (Exception ioe) { + throw new MTLog.Fatal(ioe, "I/O Error while writing trip stops file!"); + } + } + @NotNull private static Pair, Pair> dumpRDSStops(@Nullable MSpec mSpec, @NotNull String fileBase, @@ -683,12 +758,12 @@ private static Pair dumpScheduleServiceDates(@NotNull GAgencyT if (F_PRE_FILLED_DB) { FileUtils.deleteIfExist(new File(rawDirF, fileBase + GTFS_SCHEDULE_SERVICE_DATES)); // migration from src/main/res/raw to data } - File file = new File(F_PRE_FILLED_DB ? dataDirF : rawDirF, fileBase + GTFS_SCHEDULE_SERVICE_DATES); + final File file = new File(F_PRE_FILLED_DB ? dataDirF : rawDirF, fileBase + GTFS_SCHEDULE_SERVICE_DATES); + boolean empty; FileUtils.deleteIfExist(file); // delete previous - BufferedWriter ow = null; if (deleteAll) return minMaxDates; - try { - ow = new BufferedWriter(new FileWriter(file)); + try (BufferedWriter ow = new BufferedWriter(new FileWriter(file))) { + empty = true; MTLog.logPOINT(); // LOG Integer minDate = null, maxDate = null; Statement dbStatement = null; @@ -698,8 +773,17 @@ private static Pair dumpScheduleServiceDates(@NotNull GAgencyT dbStatement = dbConnection.createStatement(); sqlInsert = GTFSCommons.getT_SERVICE_DATES_SQL_INSERT(); } + MServiceDate lastServiceDate = null; for (MServiceDate mServiceDate : mSpec.getServiceDates()) { - final String serviceDatesInsert = mServiceDate.toFile(gAgencyTools); + if (!FeatureFlags.F_EXPORT_FLATTEN_SERVICE_DATES || !mServiceDate.isSameServiceId(lastServiceDate)) { + lastServiceDate = null; + if (!empty) { + ow.write(Constants.NEW_LINE); + } + } else { + ow.write(SQLUtils.COLUMN_SEPARATOR); + } + final String serviceDatesInsert = mServiceDate.toFile(gAgencyTools, lastServiceDate); if (F_PRE_FILLED_DB) { SQLUtils.executeUpdate( dbStatement, @@ -707,13 +791,21 @@ private static Pair dumpScheduleServiceDates(@NotNull GAgencyT ); } ow.write(serviceDatesInsert); - ow.write(Constants.NEW_LINE); + empty = false; if (minDate == null || minDate > mServiceDate.getCalendarDate()) { minDate = mServiceDate.getCalendarDate(); } if (maxDate == null || maxDate.doubleValue() < mServiceDate.getCalendarDate()) { maxDate = mServiceDate.getCalendarDate(); } + if (FeatureFlags.F_EXPORT_FLATTEN_SERVICE_DATES) { + lastServiceDate = mServiceDate; + } + } + if (empty) { + FileUtils.delete(file); + } else { + ow.write(Constants.NEW_LINE); // GIT convention for easier diff (ELSE unchanged line might appear as changed when not) } if (F_PRE_FILLED_DB) { SQLUtils.setAutoCommit(dbConnection, true); // END TRANSACTION == commit() @@ -721,8 +813,6 @@ private static Pair dumpScheduleServiceDates(@NotNull GAgencyT minMaxDates = new Pair<>(minDate, maxDate); } catch (Exception ioe) { throw new MTLog.Fatal(ioe, "I/O Error while writing service dates file!"); - } finally { - CloseableUtils.closeQuietly(ow); } return minMaxDates; } @@ -779,44 +869,43 @@ private static void dumpScheduleStops(@NotNull GAgencyTools gAgencyTools, } mStopScheduleMap.get(schedule.getStopId()).add(schedule); } - BufferedWriter ow = null; + // BufferedWriter ow = null; int fw = 0; for (Integer stopId : mStopScheduleMap.keySet()) { - try { - mStopSchedules = mStopScheduleMap.get(stopId); - Collections.sort(mStopSchedules); // DB sort uses IntId instead of id string - if (!mStopSchedules.isEmpty()) { - fileName = fileBaseScheduleStop + stopId; - file = new File(rawDirF, fileName); - empty = true; - ow = new BufferedWriter(new FileWriter(file)); - if (fw++ % FILE_WRITER_LOG == 0) { // LOG - MTLog.logPOINT(); // LOG - } // LOG - MSchedule lastSchedule = null; - for (MSchedule mSchedule : mStopSchedules) { - if (!mSchedule.isSameServiceAndDirection(lastSchedule)) { - lastSchedule = null; - if (!empty) { - ow.write(Constants.NEW_LINE); - } - } else { - ow.write(SQLUtils.COLUMN_SEPARATOR); + mStopSchedules = mStopScheduleMap.get(stopId); + Collections.sort(mStopSchedules); // DB sort uses IntId instead of id string + if (mStopSchedules.isEmpty()) { + continue; + } + fileName = fileBaseScheduleStop + stopId; + file = new File(rawDirF, fileName); + try (BufferedWriter ow = new BufferedWriter(new FileWriter(file))) { + empty = true; + if (fw++ % FILE_WRITER_LOG == 0) { // LOG + MTLog.logPOINT(); // LOG + } // LOG + MSchedule lastStopSchedule = null; + for (MSchedule mStopSchedule : mStopSchedules) { + if (!mStopSchedule.isSameServiceAndDirection(lastStopSchedule)) { + lastStopSchedule = null; + if (!empty) { + ow.write(Constants.NEW_LINE); } - ow.write(mSchedule.toFile(gAgencyTools, lastSchedule)); - empty = false; - lastSchedule = mSchedule; - } - if (empty) { - FileUtils.delete(file); } else { - ow.write(Constants.NEW_LINE); // GIT convention for easier diff (ELSE unchanged line might appear as changed when not) + ow.write(SQLUtils.COLUMN_SEPARATOR); } + ow.write(mStopSchedule.toFile(gAgencyTools, lastStopSchedule)); + empty = false; + lastStopSchedule = mStopSchedule; } + if (empty) { + FileUtils.delete(file); + } else { + ow.write(Constants.NEW_LINE); // GIT convention for easier diff (ELSE unchanged line might appear as changed when not) + } + } catch (IOException ioe) { throw new MTLog.Fatal(ioe, "I/O Error while writing schedule file for stop '%s'!", stopId); - } finally { - CloseableUtils.closeQuietly(ow); } } } diff --git a/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt b/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt index e3704e1..d74a3f0 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt @@ -96,9 +96,16 @@ data class MSchedule( fun toFile(agencyTools: GAgencyTools, lastSchedule: MSchedule?) = buildList { if (lastSchedule == null) { // NEW - add(MServiceIds.convert(agencyTools.cleanServiceId(_serviceId))) - // no route ID, just for file split - add(directionId.toString()) + // no stop ID, just for file split + if (FeatureFlags.F_EXPORT_SCHEDULE_SORTED_BY_ROUTE_DIRECTION) { + // no route ID, just for logs + add(directionId.toString()) + add(MServiceIds.convert(agencyTools.cleanServiceId(_serviceId))) + } else { + add(MServiceIds.convert(agencyTools.cleanServiceId(_serviceId))) + // no route ID, just for logs + add(directionId.toString()) + } } val lastDeparture = if (FeatureFlags.F_SCHEDULE_IN_MINUTES) { lastSchedule?.departure?.div(100)?.times(100) @@ -136,10 +143,9 @@ data class MSchedule( add(accessible.toString()) }.joinToString(SQLUtils.COLUMN_SEPARATOR) - fun isSameServiceAndDirection(lastSchedule: MSchedule?): Boolean { - return lastSchedule?.serviceIdInt == serviceIdInt + fun isSameServiceAndDirection(lastSchedule: MSchedule?) = + lastSchedule?.serviceIdInt == serviceIdInt && lastSchedule.directionId == directionId - } fun isSameServiceRDSDeparture(ts: MSchedule): Boolean { if (ts.serviceIdInt != serviceIdInt) { @@ -159,16 +165,21 @@ data class MSchedule( return true } - override fun compareTo(other: MSchedule): Int { - // sort by route_id => service_id => direction_id => stop_id => departure - return when { - routeId != other.routeId -> routeId.compareTo(other.routeId) - serviceIdInt != other.serviceIdInt -> _serviceId.compareTo(other._serviceId) - directionId != other.directionId -> directionId.compareTo(other.directionId) - stopId != other.stopId -> stopId - other.stopId - else -> departure - other.departure - } - } + override fun compareTo(other: MSchedule) = + if (FeatureFlags.F_EXPORT_SCHEDULE_SORTED_BY_ROUTE_DIRECTION) compareBy( + MSchedule::routeId, + MSchedule::directionId, + MSchedule::stopId, + MSchedule::_serviceId, + MSchedule::departure + ).compare(this, other) + else compareBy( + MSchedule::routeId, + MSchedule::_serviceId, + MSchedule::directionId, + MSchedule::stopId, + MSchedule::departure + ).compare(this, other) companion object { const val ROUTE_ID = "route_id" diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt index 3d9883e..efed408 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt @@ -1,6 +1,7 @@ package org.mtransit.parser.mt.data import androidx.annotation.Discouraged +import org.mtransit.commons.FeatureFlags import org.mtransit.commons.sql.SQLUtils import org.mtransit.parser.db.SQLUtils.unquotes import org.mtransit.parser.gtfs.GAgencyTools @@ -12,7 +13,7 @@ data class MServiceDate( val serviceIdInt: Int, val calendarDate: Int, val exceptionType: Int, -) : Comparable { +) { private constructor( serviceIdInt: Int, @@ -31,18 +32,20 @@ data class MServiceDate( private val _serviceId: String get() = GIDs.getString(serviceIdInt) - override fun compareTo(other: MServiceDate): Int = compareBy( - MServiceDate::calendarDate, - MServiceDate::_serviceId, - MServiceDate::exceptionType, - ).compare(this, other) - - fun toFile(agencyTools: GAgencyTools) = buildList { - add(MServiceIds.convert(agencyTools.cleanServiceId(_serviceId))) + /** + * see [org.mtransit.commons.GTFSCommons.T_SERVICE_DATES_SQL_INSERT] + */ + fun toFile(agencyTools: GAgencyTools, lastServiceDate: MServiceDate? = null) = buildList { + if (!FeatureFlags.F_EXPORT_FLATTEN_SERVICE_DATES || lastServiceDate == null) { // new + add(MServiceIds.convert(agencyTools.cleanServiceId(_serviceId))) + } add(calendarDate.toString()) add(exceptionType.toString()) }.joinToString(SQLUtils.COLUMN_SEPARATOR) + fun isSameServiceId(other: MServiceDate?) = + this.serviceIdInt == other?.serviceIdInt + @Suppress("unused") fun toStringPlus(): String { return toString() + @@ -63,6 +66,22 @@ data class MServiceDate( ) companion object { + + @JvmStatic + val COMPARATOR_BY_CALENDAR_DATE = compareBy( + MServiceDate::calendarDate, + MServiceDate::_serviceId, + MServiceDate::exceptionType, + ) + + @JvmStatic + val COMPARATOR_FOR_FILE = if (FeatureFlags.F_EXPORT_FLATTEN_SERVICE_DATES) + compareBy( + MServiceDate::_serviceId, + MServiceDate::calendarDate, + MServiceDate::exceptionType, + ) else COMPARATOR_BY_CALENDAR_DATE + @Suppress("unused") @JvmStatic fun toStringPlus(serviceDates: Iterable): String { diff --git a/src/main/java/org/mtransit/parser/mt/data/MSpec.kt b/src/main/java/org/mtransit/parser/mt/data/MSpec.kt index b6af52e..9984c97 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MSpec.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MSpec.kt @@ -15,6 +15,7 @@ data class MSpec( val routes: List, val directions: List, val directionStops: List, + val trips: List, val serviceDates: List, val routeFrequencies: TreeMap>, val firstTimestamp: Long, diff --git a/src/main/java/org/mtransit/parser/mt/data/MTrip.kt b/src/main/java/org/mtransit/parser/mt/data/MTrip.kt new file mode 100644 index 0000000..6a8147f --- /dev/null +++ b/src/main/java/org/mtransit/parser/mt/data/MTrip.kt @@ -0,0 +1,67 @@ +package org.mtransit.parser.mt.data + +import androidx.annotation.Discouraged +import org.mtransit.commons.FeatureFlags +import org.mtransit.commons.sql.SQLUtils +import org.mtransit.parser.gtfs.GAgencyTools +import org.mtransit.parser.gtfs.data.GIDs +import org.mtransit.parser.gtfs.data.GTrip + +data class MTrip( + val routeId: Long, + val directionId: Long, + val serviceIdInt: Int, + val tripIdInt: Int, +) : Comparable { + + @Suppress("unused") + @get:Discouraged(message = "Not memory efficient") + val serviceId: String get() = _serviceId + + private val _serviceId: String + get() = GIDs.getString(serviceIdInt) + + @Suppress("unused") + @get:Discouraged(message = "Not memory efficient") + val tripId: String get() = _tripId + + private val _tripId: String + get() = GIDs.getString(tripIdInt) + + fun toStringPlus() = + "${toString()}+(serviceId:$_serviceId)+(_tripId:$tripId)" + + /** + * see [org.mtransit.commons.GTFSCommons.T_TRIP_SQL_INSERT] + */ + fun toFile(agencyTools: GAgencyTools, lastTrip: MTrip? = null) = buildList { + if (!FeatureFlags.F_EXPORT_TRIP_ID) return@buildList + if (lastTrip == null) { // NEW + add(routeId.toString()) + add(directionId.toString()) + add(MServiceIds.convert(agencyTools.cleanServiceId(_serviceId))) + } + add(MTripIds.convert(_tripId)) + }.joinToString(SQLUtils.COLUMN_SEPARATOR) + + fun isSameRouteDirectionService(other: MTrip?) = + routeId == other?.routeId && directionId == other.directionId && serviceIdInt == other.serviceIdInt + + override fun compareTo(other: MTrip): Int { + // sort by route_id => service_id => direction_id => trip_id + return when { + routeId != other.routeId -> routeId.compareTo(other.routeId) + directionId != other.directionId -> directionId.compareTo(other.directionId) + serviceIdInt != other.serviceIdInt -> _serviceId.compareTo(other._serviceId) + tripIdInt != other.tripIdInt -> tripIdInt.compareTo(other.tripIdInt) + else -> 0 + } + } + + companion object { + + @JvmStatic + fun from(mRouteId: Long, mDirectionId: Long, gTrip: GTrip) = + MTrip(mRouteId, mDirectionId, gTrip.serviceIdInt, gTrip.tripIdInt) + } +} From 634a5c19f2a4bacd2e022df19a0f904c2c1afab0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Thu, 4 Dec 2025 15:37:53 -0500 Subject: [PATCH 2/4] wip --- .../java/org/mtransit/parser/db/DBUtils.kt | 7 ++++--- .../java/org/mtransit/parser/mt/data/MTrip.kt | 19 ++++++++----------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/mtransit/parser/db/DBUtils.kt b/src/main/java/org/mtransit/parser/db/DBUtils.kt index de1d893..7451b5c 100644 --- a/src/main/java/org/mtransit/parser/db/DBUtils.kt +++ b/src/main/java/org/mtransit/parser/db/DBUtils.kt @@ -416,6 +416,7 @@ object DBUtils { } } + @Suppress("AssignedValueIsNeverRead") @JvmStatic fun selectSchedules( serviceIdInt: Int? = null, @@ -491,7 +492,7 @@ object DBUtils { query += " ${MSchedule.ARRIVAL} = $arrival" } departure?.let { - query += if (whereAdded) " AND" else " WHERE" + query += if (whereAdded) " AND" else " WHERE"; whereAdded = true query += " ${MSchedule.DEPARTURE} = $departure" } query += " ORDER BY " + @@ -531,7 +532,7 @@ object DBUtils { } } - @Suppress("unused") + @Suppress("unused", "AssignedValueIsNeverRead") @JvmStatic fun deleteSchedules( serviceIdInt: Int? = null, @@ -564,7 +565,7 @@ object DBUtils { query += " ${MSchedule.ARRIVAL} = $arrival" } departure?.let { - query += if (whereAdded) " AND" else " WHERE" + query += if (whereAdded) " AND" else " WHERE"; whereAdded = true query += " ${MSchedule.DEPARTURE} = $departure" } deleteCount++ diff --git a/src/main/java/org/mtransit/parser/mt/data/MTrip.kt b/src/main/java/org/mtransit/parser/mt/data/MTrip.kt index 6a8147f..d4d70d4 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MTrip.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MTrip.kt @@ -29,7 +29,7 @@ data class MTrip( get() = GIDs.getString(tripIdInt) fun toStringPlus() = - "${toString()}+(serviceId:$_serviceId)+(_tripId:$tripId)" + "${toString()}+(serviceId:$_serviceId)+(_tripId:$_tripId)" /** * see [org.mtransit.commons.GTFSCommons.T_TRIP_SQL_INSERT] @@ -47,16 +47,13 @@ data class MTrip( fun isSameRouteDirectionService(other: MTrip?) = routeId == other?.routeId && directionId == other.directionId && serviceIdInt == other.serviceIdInt - override fun compareTo(other: MTrip): Int { - // sort by route_id => service_id => direction_id => trip_id - return when { - routeId != other.routeId -> routeId.compareTo(other.routeId) - directionId != other.directionId -> directionId.compareTo(other.directionId) - serviceIdInt != other.serviceIdInt -> _serviceId.compareTo(other._serviceId) - tripIdInt != other.tripIdInt -> tripIdInt.compareTo(other.tripIdInt) - else -> 0 - } - } + override fun compareTo(other: MTrip) = + compareBy( + MTrip::routeId, + MTrip::directionId, + MTrip::_serviceId, + MTrip::_tripId, + ).compare(this, other) companion object { From 0582816e4c9670dbb82d594e90878399c68854a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Thu, 4 Dec 2025 15:46:37 -0500 Subject: [PATCH 3/4] fix --- src/test/java/org/mtransit/parser/mt/data/MServiceDateTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/mtransit/parser/mt/data/MServiceDateTest.kt b/src/test/java/org/mtransit/parser/mt/data/MServiceDateTest.kt index c56cef3..532c60e 100644 --- a/src/test/java/org/mtransit/parser/mt/data/MServiceDateTest.kt +++ b/src/test/java/org/mtransit/parser/mt/data/MServiceDateTest.kt @@ -36,7 +36,7 @@ class MServiceDateTest { serviceDates.shuffle() val listSize = serviceDates.size - val result = serviceDates.sorted(); + val result = serviceDates.sortedWith(MServiceDate.COMPARATOR_BY_CALENDAR_DATE); assertEquals(listSize, result.size) var idx = 0 From 92a1e6eb49b2cd29f5b7e721a1cd710eadfd137b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Fri, 5 Dec 2025 11:26:06 -0500 Subject: [PATCH 4/4] fix --- src/main/java/org/mtransit/parser/mt/data/MSchedule.kt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt b/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt index d74a3f0..2ad0007 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt @@ -118,11 +118,13 @@ data class MSchedule( add((departure - lastDeparture).toString()) } if (FeatureFlags.F_EXPORT_TRIP_ID) { - var arrivalDiff = (departure - arrival).takeIf { it > MIN_ARRIVAL_DIFF_IN_HH_MM_SS } - if (FeatureFlags.F_SCHEDULE_IN_MINUTES) { - arrivalDiff = arrivalDiff?.div(100) // truncates the time to an minute that is closer to 0 + if (FeatureFlags.F_EXPORT_ARRIVAL_W_TRIP_ID) { + var arrivalDiff = (departure - arrival).takeIf { it > MIN_ARRIVAL_DIFF_IN_HH_MM_SS } + if (FeatureFlags.F_SCHEDULE_IN_MINUTES) { + arrivalDiff = arrivalDiff?.div(100) // truncates the time to an minute that is closer to 0 + } + add(arrivalDiff?.toString().orEmpty()) } - add(arrivalDiff?.toString().orEmpty()) add(MTripIds.convert(_tripId)) } if (headsignType == MDirection.HEADSIGN_TYPE_NO_PICKUP) {