Dev flex merge

configurations/test/server.yml.tmp

@@ -33,7 +33,7 @@ modules:
   gtfsapi:
     enabled: true
     load_on_fetch: false
-    # use_extension: mtc
+    use_extension: mtc
     # update_frequency: 30 # in seconds
   manager:
     normalizeFieldTransformation:
@@ -74,8 +74,12 @@ extensions:
     enabled: true
     rtd_api: http://localhost:9876/
     s3_bucket: bucket-name
+    ### Uncomment and edit if the S3 region of the bucket above is different than application.data.s3_region.
+    # s3_region: us-west-2
     s3_prefix: waiting/
     s3_download_prefix: waiting/
+    ### Uncomment and edit if separate credentials are needed to access the bucket above.
+    # s3_credentials_file: ~/.aws/credentials
   transitland:
     enabled: true
     api: https://transit.land/api/v1/feeds

pom.xml

@@ -272,7 +272,7 @@
             <groupId>com.github.ibi-group</groupId>
             <artifactId>gtfs-lib</artifactId>
             <!-- Latest dev-flex build on jitpack.io -->
-            <version>dbbf3d166cf31fc9dae490ef4b8db42ec436f743</version>
+            <version>9a3ac87a9fdb96a1516e5badf3bca6071fb6aca7</version>
             <!-- Exclusions added in order to silence SLF4J warnings about multiple bindings:
                 http://www.slf4j.org/codes.html#multiple_bindings
             -->

src/main/java/com/conveyal/datatools/common/utils/aws/S3Utils.java

@@ -22,6 +22,7 @@
 import java.io.IOException;
 import java.net.URL;
 import java.util.Date;
+import java.util.List;
 import java.util.UUID;
 
 import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt;
@@ -40,41 +41,21 @@ public class S3Utils {
     public static final String DEFAULT_BUCKET;
     public static final String DEFAULT_BUCKET_GTFS_FOLDER = "gtfs/";
 
+    public static final String APP_DATA_S3_REGION = "application.data.s3_region";
+
+    public static final String APP_DATA_CREDS_FILE = "application.data.s3_credentials_file";
+
     static {
         // Placeholder variables need to be used before setting the final variable to make sure initialization occurs
-        AmazonS3 tempS3Client = null;
         AWSCredentialsProvider tempS3CredentialsProvider = null;
         String tempGtfsS3Bucket = null;
         S3ClientManagerImpl tempS3ClientManager = null;
 
         // Only configure s3 if the config requires doing so
         if (DataManager.useS3 || hasConfigProperty("modules.gtfsapi.use_extension")) {
-            try {
-                AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();
-                String credentialsFile = DataManager.getConfigPropertyAsText("application.data.s3_credentials_file");
-                tempS3CredentialsProvider = credentialsFile != null ?
-                    new ProfileCredentialsProvider(credentialsFile, "default") :
-                    new DefaultAWSCredentialsProviderChain(); // default credentials providers, e.g. IAM role
-                builder.withCredentials(tempS3CredentialsProvider);
-
-                // If region configuration string is provided, use that.
-                // Otherwise defaults to value provided in ~/.aws/config
-                String region = DataManager.getConfigPropertyAsText("application.data.s3_region");
-                if (region != null) {
-                    builder.withRegion(region);
-                }
-                tempS3Client = builder.build();
-            } catch (Exception e) {
-                LOG.error(
-                    "S3 client not initialized correctly.  Must provide config property application.data.s3_region or specify region in ~/.aws/config",
-                    e
-                );
-            }
-
-            if (tempS3Client == null) {
-                throw new IllegalArgumentException("Fatal error initializing the default s3Client");
-            }
-            tempS3ClientManager = new S3ClientManagerImpl(tempS3Client);
+            S3Wrapper build = buildS3Wrapper(List.of(APP_DATA_CREDS_FILE), List.of(APP_DATA_S3_REGION));
+            tempS3CredentialsProvider = build.credentials;
+            tempS3ClientManager = new S3ClientManagerImpl(build.s3Client);
 
             // s3 storage
             tempGtfsS3Bucket = DataManager.getConfigPropertyAsText("application.data.gtfs_s3_bucket");
@@ -89,6 +70,76 @@ public class S3Utils {
         DEFAULT_BUCKET = tempGtfsS3Bucket;
     }
 
+    public static class S3Wrapper {
+        public final AWSCredentialsProvider credentials;
+        public final AmazonS3 s3Client;
+
+        public S3Wrapper(AWSCredentialsProvider credentials, AmazonS3 s3Client) {
+            this.credentials = credentials;
+            this.s3Client = s3Client;
+        }
+    }
+
+    /**
+     * Builds an S3 client from the specified credentials and region, or fallback to defaults
+     * using the ambient IAM role.
+     * @param configCredentials The credentials file to use for the client.
+     * @param configRegions The configuration entries in the order to try to get the AWS region to apply.
+     * @return An S3 client for the provided credentials and region.
+     */
+    public static S3Wrapper buildS3Wrapper(
+        List<String> configCredentials,
+        List<String> configRegions
+    ) throws IllegalArgumentException {
+        AmazonS3 s3client = null;
+        AWSCredentialsProvider credentialsProvider = null;
+        try {
+            AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();
+            // Iterate through the credentials and stop at the first non-null returned.
+            // Default to the ambient IAM credentials, if any.
+            for (String configCred : configCredentials) {
+                String credentialsFile = DataManager.getConfigPropertyAsText(configCred);
+                if (credentialsFile != null) {
+                    try {
+                        credentialsProvider = new ProfileCredentialsProvider(credentialsFile, "default");
+                    } catch (IllegalArgumentException e) {
+                        LOG.error("Invalid credentials from {}. Trying the next one.", configCred, e);
+                    }
+                    if (credentialsProvider != null) break;
+                }
+            }
+            if (credentialsProvider == null) {
+                // default credentials providers, e.g. IAM role
+                credentialsProvider = new DefaultAWSCredentialsProviderChain();
+            }
+            builder.withCredentials(credentialsProvider);
+
+            // Iterate through the regions and stop at the first non-null returned.
+            // Otherwise defaults to value (typically provided in ~/.aws/config)
+            for (String configRegion : configRegions) {
+                String region = DataManager.getConfigPropertyAsText(configRegion);
+                if (region != null) {
+                    builder.withRegion(region);
+                    break;
+                }
+            }
+
+            s3client = builder.build();
+        } catch (Exception e) {
+            LOG.error(
+                "S3 client not initialized correctly. Must provide config property {} or specify region in ~/.aws/config",
+                configRegions,
+                e
+            );
+        }
+
+        if (s3client == null) {
+            throw new IllegalArgumentException("Fatal error initializing the default s3Client");
+        }
+
+        return new S3Wrapper(credentialsProvider, s3client);
+    }
+
     /**
      * Makes a key for an object id that is assumed to be in the default bucket's GTFS folder
      */
@@ -105,7 +156,7 @@ public static String getDefaultBucketUriForKey(String key) {
     }
 
     public static String getDefaultBucketUrlForKey(String key) {
-        return String.format("https://s3.amazonaws.com/%s/%s", DEFAULT_BUCKET, key);
+        return String.format("https://%s.s3.amazonaws.com/%s", DEFAULT_BUCKET, key);
     }
 
     /**

src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java

@@ -313,7 +313,7 @@ private String cascadeDeleteStop(Request req, Response res) {
                         int stopSequence = patternStop.getValue();
                         // Begin with the stop prior to the one deleted, unless at the beginning.
                         int beginWithSequence = (stopSequence != 0) ? stopSequence - 1 : stopSequence;
-                        tableWriter.normalizeStopTimesForPattern(patternStop.getKey(), beginWithSequence, false);
+                        tableWriter.normalizeStopTimesForPattern(patternStop.getKey(), beginWithSequence, false, false);
                     }
                 }
             }
@@ -407,8 +407,9 @@ private String updateStopTimesFromPatternStops (Request req, Response res) {
         try {
             int beginStopSequence = Integer.parseInt(req.queryParams("stopSequence"));
             boolean interpolateStopTimes = Boolean.parseBoolean(req.queryParams("interpolateStopTimes"));
+            boolean ignoreNonBlankStopTimes = Boolean.parseBoolean(req.queryParams("ignoreNonBlankStopTimes"));
             JdbcTableWriter tableWriter = new JdbcTableWriter(table, datasource, namespace);
-            int stopTimesUpdated = tableWriter.normalizeStopTimesForPattern(patternId, beginStopSequence, interpolateStopTimes);
+            int stopTimesUpdated = tableWriter.normalizeStopTimesForPattern(patternId, beginStopSequence, interpolateStopTimes, ignoreNonBlankStopTimes);
             return SparkUtils.formatJSON("updateResult", stopTimesUpdated + " stop times updated.");
         } catch (Exception e) {
             logMessageAndHalt(req, 400, "Error normalizing stop times", e);

src/main/java/com/conveyal/datatools/manager/DataManager.java

@@ -209,25 +209,36 @@ static void registerRoutes() throws IOException {
             SnapshotController.register(EDITOR_API_PREFIX);
             EditorLockController.register(EDITOR_API_PREFIX);
 
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.AREAS, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.AGENCY, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.ATTRIBUTIONS, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.CALENDAR, DataManager.GTFS_DATA_SOURCE);
             // NOTE: fare_attributes controller handles updates to nested table fare_rules.
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.FARE_ATTRIBUTES, DataManager.GTFS_DATA_SOURCE);
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.FARE_LEG_JOIN_RULES, DataManager.GTFS_DATA_SOURCE);
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.FARE_LEG_RULES, DataManager.GTFS_DATA_SOURCE);
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.FARE_MEDIAS, DataManager.GTFS_DATA_SOURCE);
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.FARE_PRODUCTS, DataManager.GTFS_DATA_SOURCE);
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.FARE_TRANSFER_RULES, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.FEED_INFO, DataManager.GTFS_DATA_SOURCE);
+
             // NOTE: Booking rules, locations and location shapes are GTFS Flex additions.
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.BOOKING_RULES, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATIONS, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATION_GROUP, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATION_GROUP_STOPS, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATION_SHAPES, DataManager.GTFS_DATA_SOURCE);
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.NETWORKS, DataManager.GTFS_DATA_SOURCE);
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.RIDER_CATEGORIES, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.ROUTES, DataManager.GTFS_DATA_SOURCE);
             // NOTE: Patterns controller handles updates to nested tables shapes, pattern stops, and frequencies.
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.PATTERNS, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.SCHEDULE_EXCEPTIONS, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.STOPS, DataManager.GTFS_DATA_SOURCE);
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.TIME_FRAMES, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.TRANSLATIONS, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.TRIPS, DataManager.GTFS_DATA_SOURCE);
+
             // TODO: Add transfers.txt controller?
         }
 
@@ -248,7 +259,7 @@ static void registerRoutes() throws IOException {
                 String extensionFeedBucket = getExtensionPropertyAsText(extensionType, "s3_bucket");
                 String extensionBucketFolder = getExtensionPropertyAsText(extensionType, "s3_download_prefix");
                 int updateFrequency = getConfigProperty("modules.gtfsapi.update_frequency").asInt();
-                if (S3Utils.DEFAULT_BUCKET != null && extensionBucketFolder != null) {
+                if (extensionFeedBucket != null && extensionBucketFolder != null) {
                     FeedUpdater.schedule(updateFrequency, extensionFeedBucket, extensionBucketFolder);
                 }
                 else LOG.warn("FeedUpdater not initialized. S3 bucket and folder not provided.");

src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java

@@ -1,6 +1,7 @@
 package com.conveyal.datatools.manager.extensions.mtc;
 
 import com.amazonaws.AmazonServiceException;
+import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.model.PutObjectRequest;
 import com.conveyal.datatools.common.utils.CloseableHttpURLConnection;
 import com.conveyal.datatools.common.utils.aws.CheckedAWSException;
@@ -31,6 +32,8 @@
 import java.util.List;
 import java.util.Map;
 
+import static com.conveyal.datatools.common.utils.aws.S3Utils.APP_DATA_CREDS_FILE;
+import static com.conveyal.datatools.common.utils.aws.S3Utils.APP_DATA_S3_REGION;
 import static com.conveyal.datatools.manager.models.ExternalFeedSourceProperty.constructId;
 import static com.mongodb.client.model.Filters.eq;
 
@@ -54,6 +57,9 @@ public class MtcFeedResource implements ExternalFeedResource {
     public static final String RESOURCE_TYPE = "MTC";
     public static final String STOP_CODE_PRIMARY_PREFIX_FIELD_NAME = "PrimaryPrefix";
     public static final String STOP_CODE_SECONDARY_PREFIXES_FIELD_NAME = "SecondaryPrefixes";
+    public static final String CONFIG_MTC_CREDENTIALS = "extensions.mtc.s3_credentials_file";
+    public static final String CONFIG_MTC_REGION = "extensions.mtc.s3_region";
+    private static S3Utils.S3Wrapper s3Wrapper;
 
     private String rtdApi, s3Bucket, s3Prefix;
 
@@ -212,7 +218,7 @@ public void feedVersionCreated(
         LOG.info("Pushing to MTC S3 Bucket: s3://{}/{}", s3Bucket, keyName);
         File file = feedVersion.retrieveGtfsFile();
         try {
-            S3Utils.getDefaultS3Client().putObject(new PutObjectRequest(s3Bucket, keyName, file));
+            getS3Client().putObject(new PutObjectRequest(s3Bucket, keyName, file));
         } catch (Exception e) {
             LOG.error("Could not upload feed version to s3.");
             e.printStackTrace();
@@ -354,4 +360,26 @@ public static List<String> getSecondaryStopCodePrefixes(Map<String, Map<String,
         String secondaryStopPrefixValue = MtcFeedResource.getFieldValue(properties, MtcFeedResource.STOP_CODE_SECONDARY_PREFIXES_FIELD_NAME);
         return (secondaryStopPrefixValue == null) ? null : List.of(secondaryStopPrefixValue.split(","));
     }
+
+    /**
+     * Builds the S3 wrapper to access the MTC feed processing bucket,
+     * which needs separate credentials from the rest of datatools.
+     */
+    static S3Utils.S3Wrapper getS3Wrapper() {
+        List<String> configRegions = List.of(CONFIG_MTC_REGION, APP_DATA_S3_REGION);
+        List<String> configCredentials = List.of(CONFIG_MTC_CREDENTIALS, APP_DATA_CREDS_FILE);
+        return S3Utils.buildS3Wrapper(configCredentials, configRegions);
+    }
+
+    /**
+     * Get the S3 client from a cached wrapper to access the MTC feed processing bucket.
+     * Note: In S3Utils, an AWSClientManager is used, however in practice, only the default client is invoked
+     * (i.e. with role=null, region=null), so the logical path returns the original s3Client, which we are doing here.
+     */
+    public static AmazonS3 getS3Client() {
+        if (s3Wrapper == null) {
+            s3Wrapper = getS3Wrapper();
+        }
+        return s3Wrapper.s3Client;
+    }
 }

src/main/java/com/conveyal/datatools/manager/jobs/FeedUpdater.java

@@ -5,7 +5,7 @@
 import com.amazonaws.services.s3.model.S3Object;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.conveyal.datatools.common.utils.aws.CheckedAWSException;
-import com.conveyal.datatools.common.utils.aws.S3Utils;
+import com.conveyal.datatools.manager.extensions.mtc.MtcFeedResource;
 import com.conveyal.datatools.manager.models.ExternalFeedSourceProperty;
 import com.conveyal.datatools.manager.models.FeedSource;
 import com.conveyal.datatools.manager.models.FeedVersion;
@@ -406,7 +406,7 @@ private FeedVersion findMatchingFeedVersion(
     ) throws AmazonServiceException, IOException, CheckedAWSException {
         String filename = keyName.split("/")[1];
         String feedId = filename.replace(".zip", "");
-        S3Object object = S3Utils.getDefaultS3Client().getObject(feedBucket, keyName);
+        S3Object object = MtcFeedResource.getS3Client().getObject(feedBucket, keyName);
         File file = new File(FeedStore.basePath, filename);
         try (
             InputStream in = object.getObjectContent();
@@ -447,17 +447,14 @@ public interface CompletedFeedRetriever {
 
     /**
      * Implements the default behavior for above interface.
+     * This method uses extensions.mtc.s3_credentials_file and s3_region, if populated,
+     * and falls back to the application's S3 credentials and region otherwise.
      */
     public class DefaultCompletedFeedRetriever implements CompletedFeedRetriever {
         @Override
         public List<S3ObjectSummary> retrieveCompletedFeeds() {
-            try {
-                ObjectListing gtfsList = S3Utils.getDefaultS3Client().listObjects(feedBucket, bucketFolder);
-                return gtfsList.getObjectSummaries();
-            } catch (CheckedAWSException e) {
-                LOG.error("Failed to list S3 Objects", e);
-                return null;
-            }
+            ObjectListing gtfsList = MtcFeedResource.getS3Client().listObjects(feedBucket, bucketFolder);
+            return gtfsList.getObjectSummaries();
         }
     }
 }