Striim Migration Service for Google Cloud Documentation

Increasing target bandwidth and avoiding backpressure

Database Writer and Spanner Writer have a Parallel Threads property that makes it easy to create multiple writer instances that can reduce or eliminate backpressure and in some situations improve performance. For example:

CREATE TARGET Oracle2PGIL_PostgresTarget1 USING DatabaseWriter ( 
  ConnectionURL: 'jdbc:postgresql://localhost:5432/webaction?stringtype=unspecified', 
  Username: 'striim', 
  Password: '*************', 
  Tables: 'HR.%:hr.%', 
  PararllelThreads:'4'
INPUT FROM Oracle2PGIL_OutputStream;

This would create four instances of Database Writer with identical settings. Each instance would run in its own thread, increasing throughput. Events are evenly distributed among the writer instances in round-robin fashion. All instances may write to all target tables.

Enabling parallel threads disables recovery. Use this option only with your initial load application, not for streaming migration.

If parallel threads may not improve performance, you may instead create multiple applications and set the Tables and/or Excluded Tables values in the sources and targets, for example:

CREATE APPLICATION InitialLoadApp1 USE EXCEPTIONSTORE TTL : '7d' ;

CREATE SOURCE InitialLoadApp_DBSource1 USING DatabaseReader  ( 
Tables: 'HR.COUNTRIES;HR.DEPARTMENTS;HR.EMPLOYEES', 
  Username: 'qatest', 
  DatabaseProviderType: 'ORACLE', 
  Password: '9PUuzS4m9LlRbTUind0vsg==', 
  FetchSize: 10000, 
  Password_encrypted: 'true', 
  QuiesceOnILCompletion: 'true', 
  ConnectionURL: 'jdbc:oracle:thin:@//192.0.2.0:1521/XE' ) 
OUTPUT TO InitialLoadApp_OutputStream1;

CREATE TARGET InitialLoadApp_PostgresTarget1 USING DatabaseWriter  ( 
StatementCacheSize: '50', 
  ConnectionRetryPolicy: 'retryInterval=30, maxRetries=3', 
  ConnectionURL: 'jdbc:postgresql://198.51.100.0:5432/HR', 
  DatabaseProviderType: 'Default', 
  BatchPolicy: 'EventCount:1000,Interval:60', 
  CheckPointTable: 'CHKPOINT', 
  Password_encrypted: 'true', 
  PreserveSourceTransactionBoundary: 'false', 
  Password: 'jsPf42jdUYqHsNei2SIiRHmqPD1mjLsQcSyYr50gkg0=', 
  Username: 'postgres', 
  Tables: 'HR.COUNTRIES,HR.COUNTRIES;HR.DEPARTMENTS,HR.DEPARTMENTS;HR.EMPLOYEES,HR.EMPLOYEES', 
  CommitPolicy: 'EventCount:1000,Interval:60' ) 
INPUT FROM InitialLoadApp_OutputStream1;

END APPLICATION InitialLoadApp1;

CREATE APPLICATION InitialLoadApp2 USE EXCEPTIONSTORE TTL : '7d' ;

CREATE SOURCE InitialLoadApp_DBSource2 USING DatabaseReader  ( 
Tables: 'HR.JOBS;HR.JOB_HISTORY;HR.LOCATIONS;HR.REGIONS', 
  Username: 'qatest', 
  DatabaseProviderType: 'ORACLE', 
  Password: '9PUuzS4m9LlRbTUind0vsg==', 
  FetchSize: 10000, 
  Password_encrypted: 'true', 
  QuiesceOnILCompletion: 'true', 
  ConnectionURL: 'jdbc:oracle:thin:@//192.0.2.0:1521/XE' ) 
OUTPUT TO InitialLoadApp_OutputStream2  ;

CREATE TARGET InitialLoadApp_PostgresTarget2 USING DatabaseWriter  ( 
StatementCacheSize: '50', 
  ConnectionRetryPolicy: 'retryInterval=30, maxRetries=3', 
  ConnectionURL: 'jdbc:postgresql://198.51.100.0:5432/HR', 
  DatabaseProviderType: 'Default', 
  BatchPolicy: 'EventCount:1000,Interval:60', 
  CheckPointTable: 'CHKPOINT', 
  Password_encrypted: 'true', 
  PreserveSourceTransactionBoundary: 'false', 
  Password: 'jsPf42jdUYqHsNei2SIiRHmqPD1mjLsQcSyYr50gkg0=', 
  Username: 'postgres',
  Tables: 'HR.JOBS,HR.JOBS;HR.JOB_HISTORY,HR.JOB_HISTORY;HR.LOCATIONS,HR.LOCATIONS;HR.REGIONS,HR.REGIONS', 
  CommitPolicy: 'EventCount:1000,Interval:60' ) 
INPUT FROM InitialLoadApp_OutputStream2;

END APPLICATION InitialLoadApp2;