Striim Migration Service for Google Cloud Documentation

Increasing source bandwidth

If your initial load application is not fast enough, or your source writes to its CDC log faster than your streaming migration application can read it, you can increase throughput by creating multiple instances of the reader.Since each instance can be run on a different core, this can increase performance

Use the Tables property to distribute tables among the readers:

  • Assign each table to only one reader.

  • When tables are related (by primary or foreign key) or to ensure transaction integrity among a set of tables, assign them all to the same reader.

  • When dividing tables among readers, distribute them according to how busy they are rather than simply by the number of tables. For example, if one table generates 50% of the entries in the CDC log, you might assign it and any related tables to one reader and all the other tables to another.

The following is a simple example of how you could modify the initial load application created following the instructions in Create and test the initial load application to use two sources:

CREATE APPLICATION InitialLoadApp USE EXCEPTIONSTORE TTL : '7d' ;

CREATE FLOW InitialLoadApp_SourceFlow;

CREATE SOURCE InitialLoadApp_DBSource1 USING DatabaseReader  ( 
Tables: 'HR.COUNTRIES;HR.DEPARTMENTS;HR.EMPLOYEES', 
  Username: 'qatest', 
  DatabaseProviderType: 'ORACLE', 
  Password: '9PUuzS4m9LlRbTUind0vsg==', 
  FetchSize: 10000, 
  Password_encrypted: 'true', 
  QuiesceOnILCompletion: 'true', 
  ConnectionURL: 'jdbc:oracle:thin:@//192.0.2.0:1521/XE' ) 
OUTPUT TO InitialLoadApp_OutputStream  ;

CREATE SOURCE InitialLoadApp_DBSource2 USING DatabaseReader  ( 
Tables: 'HR.JOBS;HR.JOB_HISTORY;HR.LOCATIONS;HR.REGIONS', 
  Username: 'qatest', 
  DatabaseProviderType: 'ORACLE', 
  Password: '9PUuzS4m9LlRbTUind0vsg==', 
  FetchSize: 10000, 
  Password_encrypted: 'true', 
  QuiesceOnILCompletion: 'true', 
  ConnectionURL: 'jdbc:oracle:thin:@//192.0.2.0:1521/XE' ) 
OUTPUT TO InitialLoadApp_OutputStream  ;

END FLOW InitialLoadApp_SourceFlow;

CREATE  TARGET InitialLoadApp_PostgresTarget1 USING DatabaseWriter  ( 
StatementCacheSize: '50', 
  ConnectionRetryPolicy: 'retryInterval=30, maxRetries=3', 
  ConnectionURL: 'jdbc:postgresql://198.51.100.0:5432/HR', 
  DatabaseProviderType: 'Default', 
  BatchPolicy: 'EventCount:1000,Interval:60', 
  CheckPointTable: 'CHKPOINT', 
  Password_encrypted: 'true', 
  PreserveSourceTransactionBoundary: 'false', 
  Password: 'jsPf42jdUYqHsNei2SIiRHmqPD1mjLsQcSyYr50gkg0=', 
  Username: 'postgres', 
  Tables: 'HR.%,HR.%', 
  CommitPolicy: 'EventCount:1000,Interval:60' ) 
INPUT FROM InitialLoadApp_OutputStream;

END APPLICATION InitialLoadApp;