Move data parsing definitions out of the client (#11)

Makes a new (dart) package called `shared`, which contains: 
- the code needed to parse through OCCT and BC data
- any data definitions for types in that data (eg, stops)
- shared utils

The client will then import types from this package. This PR also re-generates the GET_STOPS json file to include OCCT

src/client/bin/bc.dart

@@ -1,149 +0,0 @@
-import "dart:convert";
-import "dart:io";
-import "package:csv/csv.dart";
-
-/// Utils on [Map].
-extension MapUtils<K, V> on Map<K, V> {
-  /// Gets all the keys and values as 2-element records.
-	Iterable<(K, V)> get records => entries.map((entry) => (entry.key, entry.value));
-}
-
-extension on Directory {
-  String operator /(String child) => "$path/$child";
-}
-
-final serverDir = Directory("../server");
-final dataDir = Directory(serverDir / "data/BCT");
-final tripsFile = File(dataDir / "stop_times.txt");
-final routesFile = File(dataDir / "trips.txt");
-final stopsFile = File(dataDir / "stops.txt");
-final routeNamesFile = File(dataDir / "routes.txt");
-final outputFile = File(serverDir / "GET_STOPS.json");
-
-extension type TripID(String id) { }
-extension type StopID(String id) { }
-extension type RouteID(String id) { }
-
-class Stop {
-  final StopID id;
-  final double latitude;
-  final double longitude;
-  final String name;
-  final String description;
-  final String provider;
-
-  final Set<String> routes = {};
-
-  Stop({
-    required this.id,
-    required this.latitude,
-    required this.longitude,
-    required this.name,
-    required this.description,
-    required this.provider,
-  });
-
-  Map<String, dynamic> toJson() => {
-    "id": id.id,
-    "name": name,
-    "description": description,
-    "latitude": latitude,
-    "longitude": longitude,
-    "provider": provider,
-    "routes": routes.toList(),
-  };
-}
-
-final converter = CsvCodec(shouldParseNumbers: false).decoder;
-
-Future<Map<TripID, Set<StopID>>> getTrips() async {
-  final contents = await tripsFile.readAsString();
-  final csv = converter.convert(contents);
-  final result = <TripID, Set<StopID>>{};
-  for (final row in csv.skip(1)) {
-    final tripId = TripID(row[0]);
-    final stopId = StopID(row[3]);
-    result[tripId] ??= <StopID>{};
-    result[tripId]!.add(stopId);
-  }
-  return result;
-}
-
-Future<Map<TripID, RouteID>> getRoutes() async {
-  final contents = await routesFile.readAsString();
-  final csv = converter.convert(contents);
-  final result = <TripID, RouteID>{};
-  for (final row in csv.skip(1)) {
-    final routeID = RouteID(row[0]);
-    final tripID = TripID(row[2]);
-    result[tripID] = routeID;
-  }
-  return result;
-}
-
-Future<Map<RouteID, String>> getRouteNames() async {
-  final contents = await routeNamesFile.readAsString();
-  final csv = converter.convert(contents);
-  return {
-    for (final row in csv.skip(1))
-      RouteID(row[0]): row[3],
-  };
-}
-
-Future<Map<StopID, Stop>> getStops() async {
-  final contents = await stopsFile.readAsString();
-  final csv = converter.convert(contents);
-  final result = <StopID, Stop>{};
-  for (final row in csv.skip(1)) {
-    final stopID = StopID(row[0]);
-    final name = row[2];
-    final description = row[3];
-    final latitude = double.parse(row[4]);
-    final longitude = double.parse(row[5]);
-    final stop = Stop(
-      id: stopID,
-      name: name,
-      description: description,
-      latitude: latitude,
-      longitude: longitude,
-      provider: "BC Transit",
-    );
-    result[stopID] = stop;
-  }
-  return result;
-}
-
-void findRoutesForStops({
-  required Iterable<Stop> stops,
-  required Map<TripID, Set<StopID>> trips,
-  required Map<TripID, RouteID> routes,
-  required Map<RouteID, String> routeNames,
-}) {
-  for (final stop in stops) {
-    for (final (tripID, trip) in trips.records) {
-      if (!trip.contains(stop.id)) continue;
-      final routeID = routes[tripID]!;
-      final routeName = routeNames[routeID]!;
-      stop.routes.add(routeName);
-    }
-  }
-}
-
-Future<void> outputJson(Iterable<Stop> stops) async {
-  final result = [
-    for (final stop in stops)
-      stop.toJson(),
-  ];
-  const encoder = JsonEncoder.withIndent("  ");
-  final contents = encoder.convert(result);
-  await outputFile.writeAsString(contents);
-}
-
-void main() async {
-  final trips = await getTrips();
-  final routes = await getRoutes();
-  final stops = await getStops();
-  final routeNames = await getRouteNames();
-  findRoutesForStops(stops: stops.values, trips: trips, routes: routes, routeNames: routeNames);
-  await outputJson(stops.values);
-}

src/client/lib/data.dart

@@ -2,4 +2,5 @@ export "src/data/utils.dart";
 export "src/data/trip.dart";
 export "src/data/path.dart";
 export "src/data/polyline.dart";
-export "src/data/stop.dart";
+
+export "package:shared/data.dart";

src/client/lib/src/data/path.dart

@@ -1,5 +1,5 @@
 import "trip.dart";
-import "utils.dart";
+import "package:shared/data.dart";
 
 class PathStep {
   final PathTrip trip;

src/client/lib/src/data/stop.dart

@@ -1,20 +0,0 @@
-import "utils.dart";
-
-extension type StopID(String id) { }
-
-class Stop {
-  final StopID id;
-  final String name;
-  final String description;
-  final Coordinates coordinates;
-  final String provider;
-  final List<String> routes;
-
-  Stop.fromJson(Json json) :
-    id = StopID(json["id"]),
-    name = json["name"],
-    description = json["description"],
-    coordinates = (lat: json["latitude"], long: json["longitude"]),
-    provider = json["provider"],
-    routes = (json["routes"] as List).cast<String>();
-}

src/client/lib/src/data/trip.dart

@@ -1,4 +1,4 @@
-import "utils.dart";
+import "package:shared/data.dart";
 
 import "package:flutter/material.dart" show TimeOfDay;
 

src/client/lib/src/data/utils.dart

@@ -1,35 +1,6 @@
 import "package:google_maps_flutter/google_maps_flutter.dart";
 
-/// A JSON object
-typedef Json = Map<String, dynamic>;
-
-typedef FromJson<T> = T Function(Json);
-
-typedef Coordinates = ({double lat, double long});
-
-/// Utils on [Map].
-extension MapUtils<K, V> on Map<K, V> {
-  /// Gets all the keys and values as 2-element records.
-	Iterable<(K, V)> get records => entries.map((entry) => (entry.key, entry.value));
-}
-
-/// Zips two lists, like Python
-Iterable<(E1, E2)> zip<E1, E2>(List<E1> list1, List<E2> list2) sync* {
-  if (list1.length != list2.length) throw ArgumentError("Trying to zip lists of different lengths");
-  for (var index = 0; index < list1.length; index++) {
-    yield (list1[index], list2[index]);
-  }
-}
-
-/// Extensions on lists
-extension ListUtils<E> on List<E> {
-  /// Iterates over a pair of indexes and elements, like Python
-  Iterable<(int, E)> get enumerate sync* {
-    for (var i = 0; i < length; i++) {
-      yield (i, this[i]);
-    }
-  }
-}
+import "package:shared/data.dart";
 
 extension CoordinatesUtils on Coordinates {
   LatLng toLatLng() => LatLng(lat, long);

src/client/lib/src/view_models/home_markers.dart

@@ -26,7 +26,7 @@ mixin HomeMarkers on ChangeNotifier {
           consumeTapEvents: true,
           infoWindow: InfoWindow(
             title: stop.name,
-            snippet: "${stop.description}\n\nRoutes: ${stop.routes.join("\n")}",
+            snippet: stop.summary,
           ),
         ),
   };

src/client/pubspec.lock

@@ -304,6 +304,13 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "2.1.0"
+  shared:
+    dependency: "direct main"
+    description:
+      path: "../shared"
+      relative: true
+    source: path
+    version: "1.0.0"
   shelf:
     dependency: transitive
     description:
@@ -422,5 +429,5 @@ packages:
     source: hosted
     version: "1.1.1"
 sdks:
-  dart: ">=3.7.0-0 <4.0.0"
+  dart: ">=3.7.0 <4.0.0"
   flutter: ">=3.27.0"

src/client/pubspec.yaml

@@ -17,6 +17,8 @@ dependencies:
   http: ^1.3.0
   polyline_tools: ^0.0.2
   web: ^1.1.1
+  shared:
+    path: ../shared
 
 dev_dependencies:
   dhttpd: ^4.1.0

src/server/data/OCCT/routes.json

@@ -1,5 +1,4 @@
 [
-    "https://binghamtonupublic.etaspot.net/service.php?service=get_routes&token=TESTING",
   {
     "id": 1,
     "name": "Westside Outbound",

src/server/index.ts

@@ -78,7 +78,7 @@ router.get(
   "/stops",
   async (_ctx) => {
     try {
-      const bytes = await Deno.readFile(`bct_stops.json`);
+      const bytes = await Deno.readFile(`GET_STOPS.json`);
       const headers = new Headers();
       json_mime_add(headers);
       cors_add(headers);

src/shared/.gitignore

@@ -0,0 +1,4 @@
+# https://dart.dev/guides/libraries/private-files
+# Created by `dart pub`
+.dart_tool/
+pubspec.lock

src/shared/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 1.0.0
+
+- Initial version.

src/shared/README.md

@@ -0,0 +1,2 @@
+A sample command-line application with an entrypoint in `bin/`, library code
+in `lib/`, and example unit test in `test/`.

src/shared/analysis_options.yaml

@@ -0,0 +1,52 @@
+# This file configures the analyzer, which statically analyzes Dart code to
+# check for errors, warnings, and lints. See the following for docs:
+# https://dart.dev/guides/language/analysis-options
+#
+# The issues identified by the analyzer are surfaced in the UI of Dart-enabled
+# IDEs (https://dart.dev/tools#ides-and-editors). The analyzer can also be
+# invoked from the command line by running `flutter analyze`.
+include: package:very_good_analysis/analysis_options.yaml  # has more lints
+
+analyzer:
+  language:
+    # Strict casts isn't helpful with null safety. It only notifies you on `dynamic`,
+    # which happens all the time in JSON.
+    #
+    # See https://github.com/dart-lang/language/blob/main/resources/type-system/strict-casts.md
+    strict-casts: false
+
+    # Don't let any types be inferred as `dynamic`.
+    #
+    # See https://github.com/dart-lang/language/blob/main/resources/type-system/strict-inference.md
+    strict-inference: true
+
+    # Don't let Dart infer the wrong type on the left side of an assignment.
+    #
+    # See https://github.com/dart-lang/language/blob/main/resources/type-system/strict-raw-types.md
+    strict-raw-types: true
+
+  exclude:
+    - lib/generated/**.dart
+    - test/**.dart
+    - example/**.dart
+
+linter:
+  rules:
+    # Rules NOT in package:very_good_analysis
+    prefer_double_quotes: true
+    prefer_expression_function_bodies: true
+
+    # Rules to be disabled from package:very_good_analysis
+    prefer_single_quotes: false  # prefer_double_quotes
+    lines_longer_than_80_chars: false  # lines should be at most 100 chars
+    sort_pub_dependencies: false  # Sort dependencies by function
+    use_key_in_widget_constructors: false  # not in Flutter apps
+    directives_ordering: false  # sort dart, then flutter, then package imports
+    always_use_package_imports: false  # not when importing sibling files
+    sort_constructors_first: false  # final properties, then constructor
+    avoid_dynamic_calls: false  # this lint takes over errors in the IDE
+    one_member_abstracts: false  # abstract classes are good for interfaces
+    cascade_invocations: false  # cascades are often harder to read
+
+    # Temporarily disabled until we are ready to document
+    public_member_api_docs: false

src/shared/bin/data.dart

@@ -0,0 +1,6 @@
+import "package:shared/generator.dart";
+
+void main() async {
+  final stops = StopGenerator();
+  await stops.generate();
+}

src/shared/lib/data.dart

@@ -0,0 +1,2 @@
+export "src/utils.dart";
+export "src/stops/stop.dart";

src/shared/lib/generator.dart

@@ -0,0 +1,2 @@
+export "src/generator_utils.dart";
+export "src/stops/generator.dart";

src/shared/lib/shared.dart

@@ -0,0 +1,2 @@
+export "src/stops/stop.dart";
+export "src/utils.dart";

src/shared/lib/src/generator_utils.dart

@@ -0,0 +1,56 @@
+import "dart:convert";
+import "dart:io";
+
+import "package:csv/csv.dart";
+import "package:shared/data.dart";
+export "package:shared/data.dart";
+
+extension type TripID(String id) {
+  TripID.fromJson(dynamic value) : id = value.toString();
+}
+
+extension type RouteID(String id) {
+  RouteID.fromJson(dynamic value) : id = value.toString();
+}
+
+typedef CsvRow = List<String>;
+
+extension DirectoryUtils on Directory {
+  String operator /(String child) => "$path/$child";
+}
+
+extension MapListUtils<K, V> on Map<K, List<V>> {
+  void addToList(K key, V value) {
+    this[key] ??= [];
+    this[key]!.add(value);
+  }
+}
+
+extension MapSetUtils<K, V> on Map<K, Set<V>> {
+  void addToSet(K key, V value) {
+    this[key] ??= {};
+    this[key]!.add(value);
+  }
+}
+
+abstract class Parser<T> {
+  Future<Iterable<T>> parse();
+}
+
+final csvConverter = CsvCodec(shouldParseNumbers: false).decoder;
+Future<Iterable<CsvRow>> readCsv(File file) async {
+  final contents = await file.readAsString();
+  final csv = csvConverter.convert(contents);
+  return csv.skip(1).map<CsvRow>((row) => row.cast<String>());
+}
+
+Future<List<Json>> readJson(File file) async {
+  final contents = await file.readAsString();
+  final json = jsonDecode(contents);
+  return (json as List).cast<Json>();
+}
+
+final serverDir = Directory("../server");
+final _dataDir = Directory(serverDir / "data");
+final occtDataDir = Directory(_dataDir / "OCCT");
+final bcDataDir = Directory(_dataDir / "BCT");

src/shared/lib/src/stops/bc.dart

@@ -0,0 +1,78 @@
+import "dart:io";
+
+import "package:csv/csv.dart";
+
+import "../generator_utils.dart";
+
+class BcStopParser extends Parser<Stop> {
+  static final tripsFile = File(bcDataDir / "stop_times.txt");
+  static final routesFile = File(bcDataDir / "trips.txt");
+  static final stopsFile = File(bcDataDir / "stops.txt");
+  static final routeNamesFile = File(bcDataDir / "routes.txt");
+
+  static final converter = CsvCodec(shouldParseNumbers: false).decoder;
+
+  Future<Map<TripID, Set<StopID>>> getTrips() async {
+    final result = <TripID, Set<StopID>>{};
+    for (final row in await readCsv(tripsFile)) {
+      result.addToSet(TripID(row[0]), StopID(row[3]));
+    }
+    return result;
+  }
+
+  Future<Map<TripID, RouteID>> getRoutes() async => {
+    for (final row in await readCsv(routesFile))
+      TripID(row[2]): RouteID(row[0]),
+  };
+
+  Future<Map<RouteID, String>> getRouteNames() async => {
+    for (final row in await readCsv(routeNamesFile))
+      RouteID(row[0]): row[3],
+  };
+
+  Future<Map<StopID, Stop>> getStops() async {
+    final result = <StopID, Stop>{};
+    for (final row in await readCsv(stopsFile)) {
+      final stopID = StopID(row[0]);
+      final name = row[2];
+      final description = row[3];
+      final latitude = double.parse(row[4]);
+      final longitude = double.parse(row[5]);
+      final stop = Stop(
+        id: stopID,
+        name: name,
+        description: description,
+        coordinates: (lat: latitude, long: longitude),
+        provider: "BC Transit",
+      );
+      result[stopID] = stop;
+    }
+    return result;
+  }
+
+  void findRoutesForStops({
+    required Iterable<Stop> stops,
+    required Map<TripID, Set<StopID>> trips,
+    required Map<TripID, RouteID> routes,
+    required Map<RouteID, String> routeNames,
+  }) {
+    for (final stop in stops) {
+      for (final (tripID, trip) in trips.records) {
+        if (!trip.contains(stop.id)) continue;
+        final routeID = routes[tripID]!;
+        final routeName = routeNames[routeID]!;
+        stop.routes.add(routeName);
+      }
+    }
+  }
+
+  @override
+  Future<Iterable<Stop>> parse() async {
+    final trips = await getTrips();
+    final routes = await getRoutes();
+    final stops = await getStops();
+    final routeNames = await getRouteNames();
+    findRoutesForStops(stops: stops.values, trips: trips, routes: routes, routeNames: routeNames);
+    return stops.values;
+  }
+}

src/shared/lib/src/stops/generator.dart

@@ -0,0 +1,28 @@
+import "dart:convert";
+import "dart:io";
+
+import "../generator_utils.dart";
+
+import "bc.dart";
+import "occt.dart";
+
+class StopGenerator {
+  static final serverDir = Directory("../server");
+  static final outputFile = File(serverDir / "GET_STOPS.json");
+
+  final Parser<Stop> bc = BcStopParser();
+  final Parser<Stop> occt = OcctStopParser();
+
+  Future<void> generate() async {
+    final bcStops = await bc.parse();
+    final occtStops = await occt.parse();
+    final stops = [...bcStops, ...occtStops];
+    final result = [
+      for (final stop in stops)
+        stop.toJson(),
+    ];
+    const encoder = JsonEncoder.withIndent("  ");
+    final contents = encoder.convert(result);
+    await outputFile.writeAsString(contents);
+  }
+}

src/shared/lib/src/stops/occt.dart

@@ -0,0 +1,51 @@
+import "dart:io";
+
+import "../generator_utils.dart";
+
+class OcctStopParser extends Parser<Stop> {
+  /// Taken from: https://binghamtonupublic.etaspot.net/service.php?service=get_stops&token=TESTING
+  static final stopsFile = File(occtDataDir / "stops.json");
+
+  /// Taken from: https://binghamtonupublic.etaspot.net/service.php?service=get_routes&token=TESTING
+  static final routesFile = File(occtDataDir / "routes.json");
+
+  Future<Map<StopID, Stop>> getStops() async {
+    final result = <StopID, Stop>{};
+    for (final stopJson in await readJson(stopsFile)) {
+      final id = StopID.fromJson(stopJson["id"]);
+      result[id] ??= Stop.fromOcctJson(stopJson);
+    }
+    return result;
+  }
+
+  Future<Map<StopID, List<RouteID>>> getRoutes() async {
+    final result = <StopID, List<RouteID>>{};
+    for (final json in await readJson(stopsFile)) {
+      final stopID = StopID.fromJson(json["id"]);
+      final routeID = RouteID.fromJson(json["rid"]);
+      result.addToList(stopID, routeID);
+    }
+    return result;
+  }
+
+  Future<Map<RouteID, String>> getRouteNames() async => {
+    for (final json in await readJson(routesFile))
+      RouteID.fromJson(json["id"]): json["name"],
+  };
+
+  @override
+  Future<Iterable<Stop>> parse() async {
+    final stops = await getStops();
+    final routes = await getRoutes();
+    final routeNames = await getRouteNames();
+    final routesToSkip = <RouteID>{RouteID("11")};  // no longer used
+    for (final (stopID, stop) in stops.records) {
+      for (final routeID in routes[stopID]!) {
+        if (routesToSkip.contains(routeID)) continue;
+        final routeName = routeNames[routeID]!;
+        stop.routes.add(routeName);
+      }
+    }
+    return stops.values;
+  }
+}

src/shared/lib/src/stops/stop.dart

@@ -0,0 +1,60 @@
+import "../utils.dart";
+
+extension type StopID(String id) {
+  StopID.fromJson(dynamic value) : id = value.toString();
+}
+
+class Stop {
+  final StopID id;
+  final String name;
+  final String? description;
+  final Coordinates coordinates;
+  final String provider;
+  final Set<String> routes;
+
+  Stop({
+    required this.id,
+    required this.name,
+    required this.description,
+    required this.coordinates,
+    required this.provider,
+  }) : routes = {};
+
+  Stop.fromJson(Json json) :
+    id = StopID(json["id"]),
+    name = json["name"],
+    description = json["description"],
+    coordinates = (lat: json["latitude"], long: json["longitude"]),
+    provider = json["provider"],
+    routes = (json["routes"] as List).cast<String>().toSet();
+
+  Stop.fromOcctJson(Json json) :
+    id = StopID.fromJson(json["id"]),
+    name = json["name"],
+    description = null,
+    coordinates = (lat: json["lat"], long: json["lng"]),
+    provider = "OCCT",
+    routes = <String>{};
+
+  Map<String, dynamic> toJson() => {
+    "id": id.id,
+    "name": name,
+    "description": description,
+    "latitude": coordinates.lat,
+    "longitude": coordinates.long,
+    "provider": provider,
+    "routes": routes.toList(),
+  };
+
+  String get summary {
+    final buffer = StringBuffer();
+    if (description != null) {
+      buffer.writeln(description);
+    }
+    if (routes.isNotEmpty) {
+      buffer.writeln("Routes:");
+      routes.forEach(buffer.writeln);
+    }
+    return buffer.toString();
+  }
+}

src/shared/lib/src/utils.dart

@@ -0,0 +1,30 @@
+/// A JSON object
+typedef Json = Map<String, dynamic>;
+
+typedef FromJson<T> = T Function(Json);
+
+typedef Coordinates = ({double lat, double long});
+
+/// Utils on [Map].
+extension MapUtils<K, V> on Map<K, V> {
+  /// Gets all the keys and values as 2-element records.
+	Iterable<(K, V)> get records => entries.map((entry) => (entry.key, entry.value));
+}
+
+/// Zips two lists, like Python
+Iterable<(E1, E2)> zip<E1, E2>(List<E1> list1, List<E2> list2) sync* {
+  if (list1.length != list2.length) throw ArgumentError("Trying to zip lists of different lengths");
+  for (var index = 0; index < list1.length; index++) {
+    yield (list1[index], list2[index]);
+  }
+}
+
+/// Extensions on lists
+extension ListUtils<E> on List<E> {
+  /// Iterates over a pair of indexes and elements, like Python
+  Iterable<(int, E)> get enumerate sync* {
+    for (var i = 0; i < length; i++) {
+      yield (i, this[i]);
+    }
+  }
+}

src/shared/pubspec.yaml

@@ -0,0 +1,17 @@
+name: shared
+description: A sample command-line application.
+version: 1.0.0
+# repository: https://github.com/my_org/my_repo
+
+environment:
+  sdk: ^3.7.0
+
+# Add regular dependencies here.
+dependencies:
+  csv: ^6.0.0
+  # path: ^1.8.0
+
+dev_dependencies:
+  lints: ^5.0.0
+  test: ^1.24.0
+  very_good_analysis: ^7.0.0