使用多个数据源的spring批处理慢速写入

xyhw6mcr  于 2021-07-13  发布在  Java
关注(0)|答案(0)|浏览(226)

我有一个spring批处理应用程序,可以将文件中的5m条记录加载到sqlserver数据库中。我有不同的国家代码不同的数据源。当我使用带有@primary注解的单个数据源时,spring批处理编写器会在5分钟内写入5m条记录。但是当我给多个带有@bean注解的数据源,并使用一个不是主要数据源的数据源将文件数据写入数据库时,性能变得非常慢,对于相同的5m记录,大约需要15分钟。有人能解释spring批处理是否与主数据源和其他数据源的行为不同吗。

@Repository
    @ComponentScan(basePackages = "com.abc.extract")
    @EnableBatchProcessing
    @EnableTransactionManagement
    @Slf4j
    public class DataSourceConfig {

        public DataSourceConfig() {

        }

        @Autowired
        private CCMConfiguration ccmConfig;

        @Autowired
        SpotlightConfig spotlight;

        private DataSource datasourcesVal = null;

        private DataSource datasourcesVal1 = null;

        private Map<String,DataSource> datasourcesMap = null;

        @Value("${app.country.code}")
        String countryCode;

        @Primary
        @Bean("batchprimary")
        public HikariDataSource hikariDataSource(@Value("${app.country.code}")
        String countryCode1) throws IllegalBlockSizeException, NoSuchPaddingException, BadPaddingException,
                NoSuchAlgorithmException, InvalidKeyException {
            System.out.println("Contrycode:"+countryCode1);
            SqlServerConfiguration dbConfiguration = ccmConfig.getSqlServerDatabaseConfig(countryCode1);
            HikariDataSource ds = null;
            try {
                final String password = dbConfiguration.getDataBasePwd();
                final String username = dbConfiguration.getDataBaseUserName();
                ds = (HikariDataSource) DataSourceBuilder.create().type(HikariDataSource.class).build();
                ds.setUsername(username);
                ds.setPassword(password);
                ds.setDriverClassName(dbConfiguration.getDataBaseDriver());
                ds.setJdbcUrl(dbConfiguration.getDataBaseUrl());
                ds.setConnectionTestQuery("SELECT 1");
                ds.setConnectionTimeout(dbConfiguration.getConnectionTimeout());
                ds.setIdleTimeout(dbConfiguration.getIdleTimeout());
                ds.setMaximumPoolSize(dbConfiguration.getHikariPoolSize());
                //ds.setMaximumPoolSize(2);
                ds.setMaxLifetime(dbConfiguration.getMaxLifetime());
                ds.setLeakDetectionThreshold(dbConfiguration.getLeakDetectionThreshold());
                ds.setPoolName(dbConfiguration.getPoolName());
                this.datasourcesVal = ds;
            } catch (Exception ex) {
                spotlight.sendNotification(ex, "Critical: Failed to establish Database connection");
                log.error(ex.getMessage());
            } finally {
                if (!Objects.nonNull(ds) || ds.isClosed()) {
                    ds.close();
                    spotlight.sendNotification(new NullPointerException("Primary data source creation failed"),
                            "Critical: Failed to establish Database connection");
                }
            }
            return ds;

        }

    @Bean("batchprimary1")
            public HikariDataSource hikariDataSource1(@Value("${app.country.code}")
            String countryCode1) throws IllegalBlockSizeException, NoSuchPaddingException, BadPaddingException,
                    NoSuchAlgorithmException, InvalidKeyException {
                System.out.pri

ntln("Contrycode:"+countryCode1);
            SqlServerConfiguration dbConfiguration = ccmConfig.getSqlServerDatabaseConfig(countryCode1);
            HikariDataSource ds = null;
            try {
                final String password = dbConfiguration.getDataBasePwd();
                final String username = dbConfiguration.getDataBaseUserName();
                ds = (HikariDataSource) DataSourceBuilder.create().type(HikariDataSource.class).build();
                ds.setUsername(username);
                ds.setPassword(password);
                ds.setDriverClassName(dbConfiguration.getDataBaseDriver());
                ds.setJdbcUrl(dbConfiguration.getDataBaseUrl());
                ds.setConnectionTestQuery("SELECT 1");
                ds.setConnectionTimeout(dbConfiguration.getConnectionTimeout());
                ds.setIdleTimeout(dbConfiguration.getIdleTimeout());
                ds.setMaximumPoolSize(dbConfiguration.getHikariPoolSize());
                //ds.setMaximumPoolSize(2);
                ds.setMaxLifetime(dbConfiguration.getMaxLifetime());
                ds.setLeakDetectionThreshold(dbConfiguration.getLeakDetectionThreshold());
                ds.setPoolName(dbConfiguration.getPoolName());
                //this.datasourcesVal = ds;
            } catch (Exception ex) {
                spotlight.sendNotification(ex, "Critical: Failed to establish Database connection");
                log.error(ex.getMessage());
            } finally {
                if (!Objects.nonNull(ds) || ds.isClosed()) {
                    ds.close();
                    spotlight.sendNotification(new NullPointerException("Primary data source creation failed"),
                            "Critical: Failed to establish Database connection");
                }
            }
            return ds;

        }

    @Autowired
        @Qualifier("batchprimary")
        public DataSource datasourcesVal;

        @Autowired
        @Qualifier("batchprimary1")
        public DataSource datasourcesVal1;
    @Bean
        @JobScope
        public Step ExtractNLoadItemOnHand(TaskExecutor taskExecutor,@Value("#{jobParameters}") Map<String, JobParameter> jobParameters)
                throws InvalidKeyException, IllegalBlockSizeException,
                NoSuchPaddingException, BadPaddingException,
                NoSuchAlgorithmException, SQLException, ValidationException,
                CalpiDataException {
            int chunkSize = ApplicationConstants.CHUNK_SIZE;
            log.info(" Extract and Load ItemOnHand Job - Started....");
            countryCode = jobParameters.get("CountryCode").toString();
            //dataSourceTemp = datasourcesMap.get(countryCode);
            return stepBuilderFactory
                    .get("READ ITEM ONHAND STEP")
                    .listener(stepListners)
                    .<AccumOnhand, AccumOnhand> chunk(chunkSize)
                    .reader(extractItemOnHand(null))
                    .processor(new ItemProcessor<AccumOnhand, AccumOnhand>() {
                        @Override
                        public AccumOnhand process(AccumOnhand accumOnhand)
                                throws Exception {
                            return accumOnhand;
                        }
                    })
                    .writer(compositeItemWriter.compositeItemWriter(Arrays
                            .asList(loadItemOnHand()))).faultTolerant()
                    .retryLimit(ApplicationConstants.RETRY_SKIP_LIMIT)
                    .retry(Exception.class).skip(Exception.class)
                    .skipLimit(ApplicationConstants.RETRY_SKIP_LIMIT)
                    .taskExecutor(taskExecutor)
                    .throttleLimit(ApplicationConstants.THROTTLE_LIMIT).build();
        }

        @Bean
        @StepScope
        public JdbcBatchItemWriter<AccumOnhand> loadItemOnHand()
                throws InvalidKeyException, IllegalBlockSizeException,
                NoSuchPaddingException, BadPaddingException,
                NoSuchAlgorithmException, ValidationException, CalpiDataException {
            try {
                //country = jobParameters.get("CountryCode");
                JdbcBatchItemWriter<AccumOnhand> writer;
                //datasource = dataSourceConfig.hikariDataSource(country.toString());
                if(countryCode.equals("MX")) {
                     writer = stepDBItemWriter.writeDBData(
                            MessageFormat.format(SQLConstants.INS_ACCUM_ONHAND,
                                    countryCode), datasourcesVal);
                } else {
                     writer = stepDBItemWriter.writeDBData(
                                MessageFormat.format(SQLConstants.INS_ACCUM_ONHAND,
                                        countryCode), datasourcesVal1);
                }

                log.info("Item on hand writer created successfully");
                return writer;
            } catch (Exception ex) {
                log.error("Load loadItemOnHand Failed {} ", ex);
                throw new ValidationException("Load loadItemOnHand Failed", ex);
            }
        }

暂无答案!

目前还没有任何答案,快来回答吧!

相关问题