From 1c7ac1fe978a7edba425a478334d13e7b7ffcaa7 Mon Sep 17 00:00:00 2001 From: OT-QX <43375919+ll076110@users.noreply.github.com> Date: Wed, 22 Feb 2023 09:25:30 +0800 Subject: [PATCH 01/41] [hotfix-#1518][mongodb] fix mongodb read data not support objectId type (#1520) Co-authored-by: OT-XY --- .../mongodb/converter/MongodbColumnConverter.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/chunjun-connectors/chunjun-connector-mongodb/src/main/java/com/dtstack/chunjun/connector/mongodb/converter/MongodbColumnConverter.java b/chunjun-connectors/chunjun-connector-mongodb/src/main/java/com/dtstack/chunjun/connector/mongodb/converter/MongodbColumnConverter.java index 6066f2cdf6..c54af13d02 100644 --- a/chunjun-connectors/chunjun-connector-mongodb/src/main/java/com/dtstack/chunjun/connector/mongodb/converter/MongodbColumnConverter.java +++ b/chunjun-connectors/chunjun-connector-mongodb/src/main/java/com/dtstack/chunjun/connector/mongodb/converter/MongodbColumnConverter.java @@ -28,6 +28,7 @@ import com.dtstack.chunjun.element.column.BytesColumn; import com.dtstack.chunjun.element.column.StringColumn; import com.dtstack.chunjun.element.column.TimestampColumn; +import com.dtstack.chunjun.util.GsonUtil; import org.apache.flink.table.data.RowData; import org.apache.flink.table.types.logical.LogicalType; @@ -43,6 +44,7 @@ import java.sql.Time; import java.util.ArrayList; import java.util.List; +import java.util.Map; /** * @author Ada Wong @@ -141,7 +143,13 @@ private MongoDeserializationConverter createMongoInternalConverter(LogicalType t return val -> new BigDecimalColumn(((Decimal128) val).bigDecimalValue()); case CHAR: case VARCHAR: - return val -> new StringColumn((String) val); + return val -> { + if (val instanceof List || val instanceof Map) { + return new StringColumn(GsonUtil.GSON.toJson(val)); + } else { + return new StringColumn(val.toString()); + } + }; case INTERVAL_YEAR_MONTH: case DATE: case TIME_WITHOUT_TIME_ZONE: From d981ced0a8d01d6ad4263f76fb2c26d73e424a7e Mon Sep 17 00:00:00 2001 From: FlechazoW <35768015+FlechazoW@users.noreply.github.com> Date: Tue, 8 Nov 2022 09:51:09 +0800 Subject: [PATCH 02/41] [Improve][License] Add apache-2.0 license check. (#1386) (cherry picked from commit 67604a0f05dbb4b998a45cc0a9010bf1e8c64cf2) --- .github/workflows/build.yml | 17 +++++++ .licenserc.yaml | 51 +++++++++++++++++++ chunjun-assembly/pom.xml | 19 +++++++ .../src/main/assembly/assembly.xml | 19 +++++++ chunjun-clients/.gitignore | 17 +++++++ chunjun-clients/pom.xml | 19 +++++++ .../src/main/resources/log4j.properties | 17 +++++++ chunjun-clients/src/main/resources/log4j2.xml | 19 +++++++ .../src/main/resources/logback.xml | 19 +++++++ .../chunjun-connector-binlog/pom.xml | 19 +++++++ .../connector/binlog/conf/BinlogConf.java | 10 ++-- .../binlog/inputformat/BinlogInputFormat.java | 6 +-- .../binlog/source/BinlogSourceFactory.java | 10 ++-- .../chunjun-connector-cassandra/pom.xml | 19 +++++++ .../chunjun-connector-clickhouse/pom.xml | 19 +++++++ .../clickhouse/util/ClickhouseUtil.java | 9 +--- .../chunjun-connector-db2/pom.xml | 19 +++++++ .../chunjun-connector-dm/pom.xml | 19 +++++++ .../connector/doris/rest/DorisLoadClient.java | 29 +++++------ .../chunjun-connector-elasticsearch6/pom.xml | 19 +++++++ .../chunjun-connector-elasticsearch7/pom.xml | 19 +++++++ .../chunjun-connector-emqx/pom.xml | 19 +++++++ .../chunjun-connector-filesystem/pom.xml | 19 +++++++ .../chunjun-connector-greenplum/pom.xml | 19 +++++++ .../chunjun-connector-hbase-1.4/pom.xml | 19 +++++++ .../chunjun-connector-hbase2/pom.xml | 19 +++++++ .../chunjun-connector-hdfs/pom.xml | 19 +++++++ .../chunjun-connector-hive/pom.xml | 19 +++++++ .../chunjun-connector-hive3/pom.xml | 19 +++++++ .../chunjun-connector-http/pom.xml | 19 +++++++ .../http/source/HttpSourceFactory.java | 10 ++-- .../chunjun-connector-iceberg/pom.xml | 19 +++++++ .../iceberg/source/IcebergSourceFactory.java | 5 +- .../chunjun-connector-influxdb/pom.xml | 19 +++++++ .../influxdb/conf/InfluxdbConfig.java | 31 +++++------ .../influxdb/conf/InfluxdbSourceConfig.java | 31 +++++------ .../influxdb/constants/InfluxdbCons.java | 31 +++++------ .../converter/InfluxdbColumnConverter.java | 31 +++++------ .../converter/InfluxdbRawTypeConverter.java | 31 +++++------ .../influxdb/source/InfluxdbInputFormat.java | 31 +++++------ .../source/InfluxdbInputFormatBuilder.java | 31 +++++------ .../influxdb/source/InfluxdbInputSplit.java | 31 +++++------ .../source/InfluxdbQuerySqlBuilder.java | 31 +++++------ .../source/InfluxdbSourceFactory.java | 31 +++++------ .../chunjun-connector-jdbc-base/pom.xml | 19 +++++++ .../exclusion/FieldNameExclusionStrategy.java | 19 +++++++ .../chunjun-connector-kafka/pom.xml | 19 +++++++ .../chunjun-connector-kudu/pom.xml | 19 +++++++ .../chunjun-connector-mysql/pom.xml | 19 +++++++ .../chunjun-connector-mysqlcdc/pom.xml | 19 +++++++ .../chunjun-connector-mysqld/pom.xml | 19 +++++++ .../chunjun-connector-oceanbase/pom.xml | 19 +++++++ .../chunjun-connector-oceanbasecdc/pom.xml | 19 +++++++ .../chunjun-connector-oracle/pom.xml | 19 +++++++ .../chunjun-connector-oraclelogminer/pom.xml | 19 +++++++ .../source/OraclelogminerSourceFactory.java | 10 ++-- .../chunjun-connector-pgwal/pom.xml | 19 +++++++ .../connector/pgwal/conf/PGWalConf.java | 5 +- .../pgwal/inputformat/PGWalInputFormat.java | 6 +-- .../pgwal/source/PgwalSourceFactory.java | 6 +-- .../chunjun-connector-postgresql/pom.xml | 19 +++++++ .../converter/PostgresqlColumnConverter.java | 19 +++++++ .../chunjun-connector-rabbitmq/pom.xml | 19 +++++++ .../chunjun-connector-redis/pom.xml | 19 +++++++ .../chunjun-connector-rocketmq/pom.xml | 19 +++++++ .../chunjun-connector-s3/pom.xml | 19 +++++++ .../chunjun-connector-saphana/pom.xml | 19 +++++++ .../chunjun-connector-socket/pom.xml | 19 +++++++ .../socket/source/SocketSourceFactory.java | 10 ++-- .../chunjun-connector-sqlserver/pom.xml | 19 +++++++ .../chunjun-connector-sqlservercdc/pom.xml | 19 +++++++ .../source/SqlservercdcSourceFactory.java | 10 ++-- .../chunjun-connector-starrocks/pom.xml | 19 +++++++ .../chunjun-connector-stream/pom.xml | 19 +++++++ .../chunjun-connector-sybase/pom.xml | 19 +++++++ .../vertica11/conf/Vertica11LookupConf.java | 29 +++++------ .../converter/Vertica11ColumnConverter.java | 29 +++++------ .../converter/Vertica11RawTypeConverter.java | 29 +++++------ .../converter/Vertica11RowConverter.java | 29 +++++------ .../vertica11/dialect/Vertica11Dialect.java | 1 + .../lookup/Vertica11LruTableFunction.java | 29 +++++------ .../options/Vertica11LookupOptions.java | 29 +++++------ .../provider/Vertica11DataSourceProvider.java | 29 +++++------ .../vertica11/sink/Vertica11OutputFormat.java | 29 +++++------ .../vertica11/sink/Vertica11SinkFactory.java | 28 +++++----- .../source/Vertica11DynamicTableSource.java | 29 +++++------ .../table/Vertica11DynamicTableFactory.java | 29 +++++------ chunjun-connectors/pom.xml | 19 +++++++ chunjun-core/.gitignore | 17 +++++++ chunjun-core/pom.xml | 19 +++++++ .../chunjun/cdc/QueuesChamberlain.java | 29 +++++------ .../com/dtstack/chunjun/cdc/worker/Chunk.java | 29 +++++------ .../chunjun/cdc/worker/ChunkSplitter.java | 29 +++++------ .../dtstack/chunjun/cdc/worker/Worker.java | 29 +++++------ .../chunjun/cdc/worker/WorkerManager.java | 29 +++++------ .../chunjun/cdc/worker/WorkerOverseer.java | 29 +++++------ .../com/dtstack/chunjun/decoder/IDecode.java | 10 ++-- .../dtstack/chunjun/decoder/JsonDecoder.java | 10 ++-- .../dtstack/chunjun/decoder/TextDecoder.java | 10 ++-- .../element/column/SqlArrayColumn.java | 31 +++++------ .../com/dtstack/chunjun/mapping/Mapping.java | 29 +++++------ .../dtstack/chunjun/mapping/MappingConf.java | 29 +++++------ .../dtstack/chunjun/mapping/NameMapping.java | 29 +++++------ .../chunjun/mapping/NameMappingFlatMap.java | 29 +++++------ .../src/main/resources/core-default.xml | 34 +++++++------ chunjun-core/src/test/resources/test_3.sql | 17 +++++++ chunjun-ddl/chunjun-ddl-base/pom.xml | 19 +++++++ chunjun-ddl/chunjun-ddl-mysql/pom.xml | 19 +++++++ chunjun-ddl/chunjun-ddl-oracle/pom.xml | 19 +++++++ chunjun-ddl/pom.xml | 19 +++++++ chunjun-dev/.env | 17 +++++++ chunjun-dev/app/Dockerfile | 17 +++++++ chunjun-dev/app/bashrc | 17 +++++++ chunjun-dev/app/clean_maven.sh | 17 +++++++ chunjun-dev/dm7/Dockerfile | 17 +++++++ chunjun-dev/dm8/Dockerfile | 17 +++++++ chunjun-dev/docker-compose.yml | 17 +++++++ chunjun-dev/kafka2.8/Dockerfile | 17 +++++++ chunjun-dev/kudu/Dockerfile | 17 +++++++ chunjun-dev/mysql5.7/Dockerfile | 17 +++++++ chunjun-dev/mysql8/Dockerfile | 17 +++++++ chunjun-dev/redis5/Dockerfile | 17 +++++++ chunjun-dev/redis6/Dockerfile | 17 +++++++ chunjun-dev/zookeeper/Dockerfile | 17 +++++++ chunjun-dirty/chunjun-dirty-log/pom.xml | 19 +++++++ .../src/main/resources/log4j.properties | 17 +++++++ .../src/main/resources/logback.xml | 19 +++++++ chunjun-dirty/chunjun-dirty-mysql/pom.xml | 19 +++++++ .../src/main/resources/flinkx_dirty_table.sql | 17 +++++++ chunjun-dirty/pom.xml | 19 +++++++ chunjun-docker/docker-build/Dockerfile | 17 +++++++ .../docker-build/docker-entrypoint.sh | 17 +++++++ chunjun-docker/pom.xml | 19 +++++++ chunjun-e2e/pom.xml | 19 +++++++ .../src/test/resources/docker/emqx/Dockerfile | 17 +++++++ .../resources/docker/flink/local/Dockerfile | 17 +++++++ .../docker/flink/standalone/Dockerfile | 17 +++++++ .../docker/flink/standalone/entrypoint.sh | 17 +++++++ .../src/test/resources/docker/ftp/Dockerfile | 17 +++++++ .../resources/docker/mysql/Mysql5Dockerfile | 17 +++++++ .../resources/docker/mysql/Mysql8Dockerfile | 17 +++++++ .../src/test/resources/docker/mysql/init.sql | 17 +++++++ .../src/test/resources/docker/oracle/init.sql | 17 +++++++ .../test/resources/docker/postgre/Dockerfile | 17 +++++++ .../test/resources/docker/postgre/init.sql | 17 +++++++ chunjun-formats/chunjun-protobuf/pom.xml | 19 +++++++ chunjun-formats/pom.xml | 19 +++++++ chunjun-local-test/pom.xml | 19 +++++++ .../src/main/resources/log4j.properties | 17 +++++++ .../src/main/resources/logback.xml | 19 +++++++ chunjun-metrics/chunjun-metrics-mysql/pom.xml | 19 +++++++ .../chunjun-metrics-prometheus/pom.xml | 19 +++++++ chunjun-metrics/chunjun-metrics-rdb/pom.xml | 19 +++++++ chunjun-metrics/pom.xml | 19 +++++++ .../chunjun-restore-common/pom.xml | 19 +++++++ chunjun-restore/chunjun-restore-mysql/pom.xml | 19 +++++++ .../src/main/resources/ddl.sql | 17 +++++++ chunjun-restore/pom.xml | 19 +++++++ pom.xml | 19 +++++++ 159 files changed, 2555 insertions(+), 616 deletions(-) create mode 100644 .licenserc.yaml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dc6ff64f94..d584d128a1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + name: Build With Mvnw on: [ push, pull_request ] diff --git a/.licenserc.yaml b/.licenserc.yaml new file mode 100644 index 0000000000..b8fa0348cb --- /dev/null +++ b/.licenserc.yaml @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +header: + license: + spdx-id: Apache-2.0 + copyright-owner: Apache Software Foundation + + paths-ignore: + - 'website/**' + - 'docs/**' + - 'docs_zh/**' + - 'chunjun-example/**' + - '.github/ISSUE_TEMPLATE' + - '.github/workflows' + - '.mvn' + - 'mvnw.cmd' + - '**/*.md' + - '**/*.MD' + - '**/*.json' + - '**/*.sql' + - '**/*.ftl' + - '**/target/**' + - '**/*.iml' + - '**/*.ini' + - '**/*.crt' + - '**/*.pem' + - '**/*.svg' + - '**/.gitignore' + - '**/LICENSE' + - '**/NOTICE' + - 'LICENSE' + - 'NOTICE' + + comment: on-failure diff --git a/chunjun-assembly/pom.xml b/chunjun-assembly/pom.xml index 6b58bcf0ce..d353c3b7d8 100644 --- a/chunjun-assembly/pom.xml +++ b/chunjun-assembly/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-assembly/src/main/assembly/assembly.xml b/chunjun-assembly/src/main/assembly/assembly.xml index 60a1ec172a..bcdf61912c 100644 --- a/chunjun-assembly/src/main/assembly/assembly.xml +++ b/chunjun-assembly/src/main/assembly/assembly.xml @@ -1,3 +1,22 @@ + + ${project.version} diff --git a/chunjun-clients/.gitignore b/chunjun-clients/.gitignore index 135190f227..c97fbf0ad9 100644 --- a/chunjun-clients/.gitignore +++ b/chunjun-clients/.gitignore @@ -1,2 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Created by .ignore support plugin (hsz.mobi) dependency-reduced-pom.xml \ No newline at end of file diff --git a/chunjun-clients/pom.xml b/chunjun-clients/pom.xml index 9e04c90dfb..69b3dabb3e 100644 --- a/chunjun-clients/pom.xml +++ b/chunjun-clients/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-clients/src/main/resources/log4j.properties b/chunjun-clients/src/main/resources/log4j.properties index d6f9c001f4..6a102657ca 100644 --- a/chunjun-clients/src/main/resources/log4j.properties +++ b/chunjun-clients/src/main/resources/log4j.properties @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ############# # 输出到控制台 ############# diff --git a/chunjun-clients/src/main/resources/log4j2.xml b/chunjun-clients/src/main/resources/log4j2.xml index 3762786453..a925daf286 100644 --- a/chunjun-clients/src/main/resources/log4j2.xml +++ b/chunjun-clients/src/main/resources/log4j2.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-clients/src/main/resources/logback.xml b/chunjun-clients/src/main/resources/logback.xml index d94a19a69b..eaf48fac4b 100644 --- a/chunjun-clients/src/main/resources/logback.xml +++ b/chunjun-clients/src/main/resources/logback.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-binlog/pom.xml b/chunjun-connectors/chunjun-connector-binlog/pom.xml index 218cedd4b0..414982e582 100644 --- a/chunjun-connectors/chunjun-connector-binlog/pom.xml +++ b/chunjun-connectors/chunjun-connector-binlog/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-binlog/src/main/java/com/dtstack/chunjun/connector/binlog/conf/BinlogConf.java b/chunjun-connectors/chunjun-connector-binlog/src/main/java/com/dtstack/chunjun/connector/binlog/conf/BinlogConf.java index c37d280f9c..d19e01cea1 100644 --- a/chunjun-connectors/chunjun-connector-binlog/src/main/java/com/dtstack/chunjun/connector/binlog/conf/BinlogConf.java +++ b/chunjun-connectors/chunjun-connector-binlog/src/main/java/com/dtstack/chunjun/connector/binlog/conf/BinlogConf.java @@ -6,15 +6,16 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package com.dtstack.chunjun.connector.binlog.conf; import com.dtstack.chunjun.conf.ChunJunCommonConf; @@ -23,11 +24,6 @@ import java.util.List; import java.util.Map; -/** - * config of binlog - * - * @author jiangbo @ 2020/1/10 - */ public class BinlogConf extends ChunJunCommonConf { public String host; diff --git a/chunjun-connectors/chunjun-connector-binlog/src/main/java/com/dtstack/chunjun/connector/binlog/inputformat/BinlogInputFormat.java b/chunjun-connectors/chunjun-connector-binlog/src/main/java/com/dtstack/chunjun/connector/binlog/inputformat/BinlogInputFormat.java index 16165b8fc9..b365b8314a 100644 --- a/chunjun-connectors/chunjun-connector-binlog/src/main/java/com/dtstack/chunjun/connector/binlog/inputformat/BinlogInputFormat.java +++ b/chunjun-connectors/chunjun-connector-binlog/src/main/java/com/dtstack/chunjun/connector/binlog/inputformat/BinlogInputFormat.java @@ -6,15 +6,16 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package com.dtstack.chunjun.connector.binlog.inputformat; import com.dtstack.chunjun.connector.binlog.conf.BinlogConf; @@ -53,7 +54,6 @@ import java.util.concurrent.locks.LockSupport; import java.util.stream.Collectors; -/** @author toutian */ public class BinlogInputFormat extends BaseRichInputFormat { protected BinlogConf binlogConf; diff --git a/chunjun-connectors/chunjun-connector-binlog/src/main/java/com/dtstack/chunjun/connector/binlog/source/BinlogSourceFactory.java b/chunjun-connectors/chunjun-connector-binlog/src/main/java/com/dtstack/chunjun/connector/binlog/source/BinlogSourceFactory.java index 71bcccfd85..729fd8ea16 100644 --- a/chunjun-connectors/chunjun-connector-binlog/src/main/java/com/dtstack/chunjun/connector/binlog/source/BinlogSourceFactory.java +++ b/chunjun-connectors/chunjun-connector-binlog/src/main/java/com/dtstack/chunjun/connector/binlog/source/BinlogSourceFactory.java @@ -6,15 +6,16 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package com.dtstack.chunjun.connector.binlog.source; import com.dtstack.chunjun.conf.SyncConf; @@ -35,11 +36,6 @@ import org.apache.flink.table.data.RowData; import org.apache.flink.table.types.logical.RowType; -/** - * @company: www.dtstack.com - * @author: toutian - * @create: 2019/7/4 - */ public class BinlogSourceFactory extends SourceFactory { private final BinlogConf binlogConf; diff --git a/chunjun-connectors/chunjun-connector-cassandra/pom.xml b/chunjun-connectors/chunjun-connector-cassandra/pom.xml index eca2cc835f..ea914c1a0f 100644 --- a/chunjun-connectors/chunjun-connector-cassandra/pom.xml +++ b/chunjun-connectors/chunjun-connector-cassandra/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-clickhouse/pom.xml b/chunjun-connectors/chunjun-connector-clickhouse/pom.xml index 5aaa2fdc13..bff541e2d2 100644 --- a/chunjun-connectors/chunjun-connector-clickhouse/pom.xml +++ b/chunjun-connectors/chunjun-connector-clickhouse/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-clickhouse/src/main/java/com/dtstack/chunjun/connector/clickhouse/util/ClickhouseUtil.java b/chunjun-connectors/chunjun-connector-clickhouse/src/main/java/com/dtstack/chunjun/connector/clickhouse/util/ClickhouseUtil.java index c4c2514584..e0047c2fc7 100644 --- a/chunjun-connectors/chunjun-connector-clickhouse/src/main/java/com/dtstack/chunjun/connector/clickhouse/util/ClickhouseUtil.java +++ b/chunjun-connectors/chunjun-connector-clickhouse/src/main/java/com/dtstack/chunjun/connector/clickhouse/util/ClickhouseUtil.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -28,11 +28,6 @@ import java.sql.Statement; import java.util.Properties; -/** - * Date: 2019/11/05 Company: www.dtstack.com - * - * @author tudou - */ public class ClickhouseUtil { private static final int MAX_RETRY_TIMES = 3; diff --git a/chunjun-connectors/chunjun-connector-db2/pom.xml b/chunjun-connectors/chunjun-connector-db2/pom.xml index bfb3d970f6..4e831969c5 100644 --- a/chunjun-connectors/chunjun-connector-db2/pom.xml +++ b/chunjun-connectors/chunjun-connector-db2/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-dm/pom.xml b/chunjun-connectors/chunjun-connector-dm/pom.xml index b7ff62f87d..dbc2a93b89 100644 --- a/chunjun-connectors/chunjun-connector-dm/pom.xml +++ b/chunjun-connectors/chunjun-connector-dm/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/DorisLoadClient.java b/chunjun-connectors/chunjun-connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/DorisLoadClient.java index 5d6a74d299..32ed058913 100644 --- a/chunjun-connectors/chunjun-connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/DorisLoadClient.java +++ b/chunjun-connectors/chunjun-connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/DorisLoadClient.java @@ -1,21 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.connector.doris.rest; diff --git a/chunjun-connectors/chunjun-connector-elasticsearch6/pom.xml b/chunjun-connectors/chunjun-connector-elasticsearch6/pom.xml index c0d3124cd3..70cd5035eb 100644 --- a/chunjun-connectors/chunjun-connector-elasticsearch6/pom.xml +++ b/chunjun-connectors/chunjun-connector-elasticsearch6/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-elasticsearch7/pom.xml b/chunjun-connectors/chunjun-connector-elasticsearch7/pom.xml index ee6dc84f1f..7fe464afc0 100644 --- a/chunjun-connectors/chunjun-connector-elasticsearch7/pom.xml +++ b/chunjun-connectors/chunjun-connector-elasticsearch7/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-emqx/pom.xml b/chunjun-connectors/chunjun-connector-emqx/pom.xml index 374e9da6c2..a3f67ec752 100644 --- a/chunjun-connectors/chunjun-connector-emqx/pom.xml +++ b/chunjun-connectors/chunjun-connector-emqx/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-filesystem/pom.xml b/chunjun-connectors/chunjun-connector-filesystem/pom.xml index 3140a1a8a1..ee7e11561f 100644 --- a/chunjun-connectors/chunjun-connector-filesystem/pom.xml +++ b/chunjun-connectors/chunjun-connector-filesystem/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-greenplum/pom.xml b/chunjun-connectors/chunjun-connector-greenplum/pom.xml index 447d123656..888d8b76e9 100644 --- a/chunjun-connectors/chunjun-connector-greenplum/pom.xml +++ b/chunjun-connectors/chunjun-connector-greenplum/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-hbase-1.4/pom.xml b/chunjun-connectors/chunjun-connector-hbase-1.4/pom.xml index 4a300e6d65..edff5828b1 100644 --- a/chunjun-connectors/chunjun-connector-hbase-1.4/pom.xml +++ b/chunjun-connectors/chunjun-connector-hbase-1.4/pom.xml @@ -1,4 +1,23 @@ + + + diff --git a/chunjun-connectors/chunjun-connector-hdfs/pom.xml b/chunjun-connectors/chunjun-connector-hdfs/pom.xml index 0adce16088..9faa33edac 100644 --- a/chunjun-connectors/chunjun-connector-hdfs/pom.xml +++ b/chunjun-connectors/chunjun-connector-hdfs/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-hive/pom.xml b/chunjun-connectors/chunjun-connector-hive/pom.xml index cd6cbe0ab5..0e32951296 100644 --- a/chunjun-connectors/chunjun-connector-hive/pom.xml +++ b/chunjun-connectors/chunjun-connector-hive/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-hive3/pom.xml b/chunjun-connectors/chunjun-connector-hive3/pom.xml index c333d4cac0..4520cd9212 100644 --- a/chunjun-connectors/chunjun-connector-hive3/pom.xml +++ b/chunjun-connectors/chunjun-connector-hive3/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-http/pom.xml b/chunjun-connectors/chunjun-connector-http/pom.xml index 321605baf2..3a32617257 100644 --- a/chunjun-connectors/chunjun-connector-http/pom.xml +++ b/chunjun-connectors/chunjun-connector-http/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/source/HttpSourceFactory.java b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/source/HttpSourceFactory.java index 371a30805e..cc48c9e58b 100644 --- a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/source/HttpSourceFactory.java +++ b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/source/HttpSourceFactory.java @@ -6,15 +6,16 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package com.dtstack.chunjun.connector.http.source; import com.dtstack.chunjun.conf.FieldConf; @@ -47,11 +48,6 @@ import java.util.Collections; import java.util.List; -/** - * @company: www.dtstack.com - * @author: shifang - * @create: 2019/7/4 - */ public class HttpSourceFactory extends SourceFactory { private final HttpRestConfig httpRestConfig; diff --git a/chunjun-connectors/chunjun-connector-iceberg/pom.xml b/chunjun-connectors/chunjun-connector-iceberg/pom.xml index 38d21fb713..a2bd6752a3 100644 --- a/chunjun-connectors/chunjun-connector-iceberg/pom.xml +++ b/chunjun-connectors/chunjun-connector-iceberg/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/source/IcebergSourceFactory.java b/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/source/IcebergSourceFactory.java index 73306d175e..87146921ba 100644 --- a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/source/IcebergSourceFactory.java +++ b/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/source/IcebergSourceFactory.java @@ -6,15 +6,16 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package com.dtstack.chunjun.connector.iceberg.source; import com.dtstack.chunjun.conf.SyncConf; diff --git a/chunjun-connectors/chunjun-connector-influxdb/pom.xml b/chunjun-connectors/chunjun-connector-influxdb/pom.xml index 06f5755ee4..476dc674d6 100644 --- a/chunjun-connectors/chunjun-connector-influxdb/pom.xml +++ b/chunjun-connectors/chunjun-connector-influxdb/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/conf/InfluxdbConfig.java b/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/conf/InfluxdbConfig.java index c391c4e186..4ca1a6a291 100644 --- a/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/conf/InfluxdbConfig.java +++ b/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/conf/InfluxdbConfig.java @@ -1,23 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * - * * * Licensed to the Apache Software Foundation (ASF) under one - * * * or more contributor license agreements. See the NOTICE file - * * * distributed with this work for additional information - * * * regarding copyright ownership. The ASF licenses this file - * * * to you under the Apache License, Version 2.0 (the - * * * "License"); you may not use this file except in compliance - * * * with the License. You may obtain a copy of the License at - * * * - * * * http://www.apache.org/licenses/LICENSE-2.0 - * * * - * * * Unless required by applicable law or agreed to in writing, software - * * * distributed under the License is distributed on an "AS IS" BASIS, - * * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * * See the License for the specific language governing permissions and - * * * limitations under the License. - * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.connector.influxdb.conf; diff --git a/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/conf/InfluxdbSourceConfig.java b/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/conf/InfluxdbSourceConfig.java index c36cb42407..0a22a30f73 100644 --- a/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/conf/InfluxdbSourceConfig.java +++ b/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/conf/InfluxdbSourceConfig.java @@ -1,23 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * - * * * Licensed to the Apache Software Foundation (ASF) under one - * * * or more contributor license agreements. See the NOTICE file - * * * distributed with this work for additional information - * * * regarding copyright ownership. The ASF licenses this file - * * * to you under the Apache License, Version 2.0 (the - * * * "License"); you may not use this file except in compliance - * * * with the License. You may obtain a copy of the License at - * * * - * * * http://www.apache.org/licenses/LICENSE-2.0 - * * * - * * * Unless required by applicable law or agreed to in writing, software - * * * distributed under the License is distributed on an "AS IS" BASIS, - * * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * * See the License for the specific language governing permissions and - * * * limitations under the License. - * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.connector.influxdb.conf; diff --git a/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/constants/InfluxdbCons.java b/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/constants/InfluxdbCons.java index 385caa559f..82076bea58 100644 --- a/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/constants/InfluxdbCons.java +++ b/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/constants/InfluxdbCons.java @@ -1,23 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * - * * * Licensed to the Apache Software Foundation (ASF) under one - * * * or more contributor license agreements. See the NOTICE file - * * * distributed with this work for additional information - * * * regarding copyright ownership. The ASF licenses this file - * * * to you under the Apache License, Version 2.0 (the - * * * "License"); you may not use this file except in compliance - * * * with the License. You may obtain a copy of the License at - * * * - * * * http://www.apache.org/licenses/LICENSE-2.0 - * * * - * * * Unless required by applicable law or agreed to in writing, software - * * * distributed under the License is distributed on an "AS IS" BASIS, - * * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * * See the License for the specific language governing permissions and - * * * limitations under the License. - * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.connector.influxdb.constants; diff --git a/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/converter/InfluxdbColumnConverter.java b/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/converter/InfluxdbColumnConverter.java index afc378c274..7ebbf2b3db 100644 --- a/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/converter/InfluxdbColumnConverter.java +++ b/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/converter/InfluxdbColumnConverter.java @@ -1,23 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * - * * * Licensed to the Apache Software Foundation (ASF) under one - * * * or more contributor license agreements. See the NOTICE file - * * * distributed with this work for additional information - * * * regarding copyright ownership. The ASF licenses this file - * * * to you under the Apache License, Version 2.0 (the - * * * "License"); you may not use this file except in compliance - * * * with the License. You may obtain a copy of the License at - * * * - * * * http://www.apache.org/licenses/LICENSE-2.0 - * * * - * * * Unless required by applicable law or agreed to in writing, software - * * * distributed under the License is distributed on an "AS IS" BASIS, - * * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * * See the License for the specific language governing permissions and - * * * limitations under the License. - * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.connector.influxdb.converter; diff --git a/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/converter/InfluxdbRawTypeConverter.java b/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/converter/InfluxdbRawTypeConverter.java index edcec16690..60b6ed514f 100644 --- a/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/converter/InfluxdbRawTypeConverter.java +++ b/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/converter/InfluxdbRawTypeConverter.java @@ -1,23 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * - * * * Licensed to the Apache Software Foundation (ASF) under one - * * * or more contributor license agreements. See the NOTICE file - * * * distributed with this work for additional information - * * * regarding copyright ownership. The ASF licenses this file - * * * to you under the Apache License, Version 2.0 (the - * * * "License"); you may not use this file except in compliance - * * * with the License. You may obtain a copy of the License at - * * * - * * * http://www.apache.org/licenses/LICENSE-2.0 - * * * - * * * Unless required by applicable law or agreed to in writing, software - * * * distributed under the License is distributed on an "AS IS" BASIS, - * * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * * See the License for the specific language governing permissions and - * * * limitations under the License. - * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.connector.influxdb.converter; diff --git a/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/source/InfluxdbInputFormat.java b/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/source/InfluxdbInputFormat.java index e550607d3b..77d597b15c 100644 --- a/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/source/InfluxdbInputFormat.java +++ b/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/source/InfluxdbInputFormat.java @@ -1,23 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * - * * * Licensed to the Apache Software Foundation (ASF) under one - * * * or more contributor license agreements. See the NOTICE file - * * * distributed with this work for additional information - * * * regarding copyright ownership. The ASF licenses this file - * * * to you under the Apache License, Version 2.0 (the - * * * "License"); you may not use this file except in compliance - * * * with the License. You may obtain a copy of the License at - * * * - * * * http://www.apache.org/licenses/LICENSE-2.0 - * * * - * * * Unless required by applicable law or agreed to in writing, software - * * * distributed under the License is distributed on an "AS IS" BASIS, - * * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * * See the License for the specific language governing permissions and - * * * limitations under the License. - * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.connector.influxdb.source; diff --git a/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/source/InfluxdbInputFormatBuilder.java b/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/source/InfluxdbInputFormatBuilder.java index ebb459cba6..68afbb585b 100644 --- a/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/source/InfluxdbInputFormatBuilder.java +++ b/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/source/InfluxdbInputFormatBuilder.java @@ -1,23 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * - * * * Licensed to the Apache Software Foundation (ASF) under one - * * * or more contributor license agreements. See the NOTICE file - * * * distributed with this work for additional information - * * * regarding copyright ownership. The ASF licenses this file - * * * to you under the Apache License, Version 2.0 (the - * * * "License"); you may not use this file except in compliance - * * * with the License. You may obtain a copy of the License at - * * * - * * * http://www.apache.org/licenses/LICENSE-2.0 - * * * - * * * Unless required by applicable law or agreed to in writing, software - * * * distributed under the License is distributed on an "AS IS" BASIS, - * * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * * See the License for the specific language governing permissions and - * * * limitations under the License. - * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.connector.influxdb.source; diff --git a/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/source/InfluxdbInputSplit.java b/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/source/InfluxdbInputSplit.java index d5fbbc967f..4a926445a5 100644 --- a/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/source/InfluxdbInputSplit.java +++ b/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/source/InfluxdbInputSplit.java @@ -1,23 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * - * * * Licensed to the Apache Software Foundation (ASF) under one - * * * or more contributor license agreements. See the NOTICE file - * * * distributed with this work for additional information - * * * regarding copyright ownership. The ASF licenses this file - * * * to you under the Apache License, Version 2.0 (the - * * * "License"); you may not use this file except in compliance - * * * with the License. You may obtain a copy of the License at - * * * - * * * http://www.apache.org/licenses/LICENSE-2.0 - * * * - * * * Unless required by applicable law or agreed to in writing, software - * * * distributed under the License is distributed on an "AS IS" BASIS, - * * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * * See the License for the specific language governing permissions and - * * * limitations under the License. - * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.connector.influxdb.source; diff --git a/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/source/InfluxdbQuerySqlBuilder.java b/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/source/InfluxdbQuerySqlBuilder.java index 7ad71cd3dc..bf7372c7c1 100644 --- a/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/source/InfluxdbQuerySqlBuilder.java +++ b/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/source/InfluxdbQuerySqlBuilder.java @@ -1,23 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * - * * * Licensed to the Apache Software Foundation (ASF) under one - * * * or more contributor license agreements. See the NOTICE file - * * * distributed with this work for additional information - * * * regarding copyright ownership. The ASF licenses this file - * * * to you under the Apache License, Version 2.0 (the - * * * "License"); you may not use this file except in compliance - * * * with the License. You may obtain a copy of the License at - * * * - * * * http://www.apache.org/licenses/LICENSE-2.0 - * * * - * * * Unless required by applicable law or agreed to in writing, software - * * * distributed under the License is distributed on an "AS IS" BASIS, - * * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * * See the License for the specific language governing permissions and - * * * limitations under the License. - * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.connector.influxdb.source; diff --git a/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/source/InfluxdbSourceFactory.java b/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/source/InfluxdbSourceFactory.java index de8f798be0..1d49c1a6ed 100644 --- a/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/source/InfluxdbSourceFactory.java +++ b/chunjun-connectors/chunjun-connector-influxdb/src/main/java/com/dtstack/chunjun/connector/influxdb/source/InfluxdbSourceFactory.java @@ -1,23 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * - * * * Licensed to the Apache Software Foundation (ASF) under one - * * * or more contributor license agreements. See the NOTICE file - * * * distributed with this work for additional information - * * * regarding copyright ownership. The ASF licenses this file - * * * to you under the Apache License, Version 2.0 (the - * * * "License"); you may not use this file except in compliance - * * * with the License. You may obtain a copy of the License at - * * * - * * * http://www.apache.org/licenses/LICENSE-2.0 - * * * - * * * Unless required by applicable law or agreed to in writing, software - * * * distributed under the License is distributed on an "AS IS" BASIS, - * * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * * See the License for the specific language governing permissions and - * * * limitations under the License. - * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.connector.influxdb.source; diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/pom.xml b/chunjun-connectors/chunjun-connector-jdbc-base/pom.xml index ba09aa5297..770caf70ee 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/pom.xml +++ b/chunjun-connectors/chunjun-connector-jdbc-base/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/exclusion/FieldNameExclusionStrategy.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/exclusion/FieldNameExclusionStrategy.java index f9f6bdc34d..54c94a4643 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/exclusion/FieldNameExclusionStrategy.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/exclusion/FieldNameExclusionStrategy.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /* * * * diff --git a/chunjun-connectors/chunjun-connector-kafka/pom.xml b/chunjun-connectors/chunjun-connector-kafka/pom.xml index aa7850816b..0d91bc7914 100644 --- a/chunjun-connectors/chunjun-connector-kafka/pom.xml +++ b/chunjun-connectors/chunjun-connector-kafka/pom.xml @@ -1,4 +1,23 @@ + + chunjun-connectors diff --git a/chunjun-connectors/chunjun-connector-kudu/pom.xml b/chunjun-connectors/chunjun-connector-kudu/pom.xml index 6eb75a9b80..6352211e35 100644 --- a/chunjun-connectors/chunjun-connector-kudu/pom.xml +++ b/chunjun-connectors/chunjun-connector-kudu/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-mysql/pom.xml b/chunjun-connectors/chunjun-connector-mysql/pom.xml index c3fd63ae1a..c5b86c6d16 100644 --- a/chunjun-connectors/chunjun-connector-mysql/pom.xml +++ b/chunjun-connectors/chunjun-connector-mysql/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-mysqlcdc/pom.xml b/chunjun-connectors/chunjun-connector-mysqlcdc/pom.xml index f9c5a90d5f..77109c0ebc 100644 --- a/chunjun-connectors/chunjun-connector-mysqlcdc/pom.xml +++ b/chunjun-connectors/chunjun-connector-mysqlcdc/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-mysqld/pom.xml b/chunjun-connectors/chunjun-connector-mysqld/pom.xml index 2dc2310c3b..2bb22c38f4 100644 --- a/chunjun-connectors/chunjun-connector-mysqld/pom.xml +++ b/chunjun-connectors/chunjun-connector-mysqld/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-oceanbase/pom.xml b/chunjun-connectors/chunjun-connector-oceanbase/pom.xml index 81f7e0d591..75124fa284 100644 --- a/chunjun-connectors/chunjun-connector-oceanbase/pom.xml +++ b/chunjun-connectors/chunjun-connector-oceanbase/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-oceanbasecdc/pom.xml b/chunjun-connectors/chunjun-connector-oceanbasecdc/pom.xml index 208323af65..93094c9394 100644 --- a/chunjun-connectors/chunjun-connector-oceanbasecdc/pom.xml +++ b/chunjun-connectors/chunjun-connector-oceanbasecdc/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-oracle/pom.xml b/chunjun-connectors/chunjun-connector-oracle/pom.xml index 7209a6192a..02c0b3ba61 100644 --- a/chunjun-connectors/chunjun-connector-oracle/pom.xml +++ b/chunjun-connectors/chunjun-connector-oracle/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-oraclelogminer/pom.xml b/chunjun-connectors/chunjun-connector-oraclelogminer/pom.xml index 0e918bb4aa..93dddf2ae3 100644 --- a/chunjun-connectors/chunjun-connector-oraclelogminer/pom.xml +++ b/chunjun-connectors/chunjun-connector-oraclelogminer/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-oraclelogminer/src/main/java/com/dtstack/chunjun/connector/oraclelogminer/source/OraclelogminerSourceFactory.java b/chunjun-connectors/chunjun-connector-oraclelogminer/src/main/java/com/dtstack/chunjun/connector/oraclelogminer/source/OraclelogminerSourceFactory.java index 7de4072f89..35e62d1e6e 100644 --- a/chunjun-connectors/chunjun-connector-oraclelogminer/src/main/java/com/dtstack/chunjun/connector/oraclelogminer/source/OraclelogminerSourceFactory.java +++ b/chunjun-connectors/chunjun-connector-oraclelogminer/src/main/java/com/dtstack/chunjun/connector/oraclelogminer/source/OraclelogminerSourceFactory.java @@ -6,15 +6,16 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package com.dtstack.chunjun.connector.oraclelogminer.source; import com.dtstack.chunjun.conf.SyncConf; @@ -37,11 +38,6 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -/** - * @company: www.dtstack.com - * @author: tudou - * @create: 2019/7/4 - */ public class OraclelogminerSourceFactory extends SourceFactory { private final LogMinerConf logMinerConf; diff --git a/chunjun-connectors/chunjun-connector-pgwal/pom.xml b/chunjun-connectors/chunjun-connector-pgwal/pom.xml index f200cb971a..e51800c654 100644 --- a/chunjun-connectors/chunjun-connector-pgwal/pom.xml +++ b/chunjun-connectors/chunjun-connector-pgwal/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-pgwal/src/main/java/com/dtstack/chunjun/connector/pgwal/conf/PGWalConf.java b/chunjun-connectors/chunjun-connector-pgwal/src/main/java/com/dtstack/chunjun/connector/pgwal/conf/PGWalConf.java index a654158f1e..7916c85f69 100644 --- a/chunjun-connectors/chunjun-connector-pgwal/src/main/java/com/dtstack/chunjun/connector/pgwal/conf/PGWalConf.java +++ b/chunjun-connectors/chunjun-connector-pgwal/src/main/java/com/dtstack/chunjun/connector/pgwal/conf/PGWalConf.java @@ -6,15 +6,16 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package com.dtstack.chunjun.connector.pgwal.conf; import com.dtstack.chunjun.conf.ChunJunCommonConf; diff --git a/chunjun-connectors/chunjun-connector-pgwal/src/main/java/com/dtstack/chunjun/connector/pgwal/inputformat/PGWalInputFormat.java b/chunjun-connectors/chunjun-connector-pgwal/src/main/java/com/dtstack/chunjun/connector/pgwal/inputformat/PGWalInputFormat.java index 15eb2bf685..c37d6e32ec 100644 --- a/chunjun-connectors/chunjun-connector-pgwal/src/main/java/com/dtstack/chunjun/connector/pgwal/inputformat/PGWalInputFormat.java +++ b/chunjun-connectors/chunjun-connector-pgwal/src/main/java/com/dtstack/chunjun/connector/pgwal/inputformat/PGWalInputFormat.java @@ -6,15 +6,16 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package com.dtstack.chunjun.connector.pgwal.inputformat; import com.dtstack.chunjun.connector.pgwal.conf.PGWalConf; @@ -48,7 +49,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; -/** */ public class PGWalInputFormat extends BaseRichInputFormat { private static final Logger LOG = LoggerFactory.getLogger(PGWalInputFormat.class); private static Thread.UncaughtExceptionHandler exceptionHandler = diff --git a/chunjun-connectors/chunjun-connector-pgwal/src/main/java/com/dtstack/chunjun/connector/pgwal/source/PgwalSourceFactory.java b/chunjun-connectors/chunjun-connector-pgwal/src/main/java/com/dtstack/chunjun/connector/pgwal/source/PgwalSourceFactory.java index 6a498965d2..eb8e6854f5 100644 --- a/chunjun-connectors/chunjun-connector-pgwal/src/main/java/com/dtstack/chunjun/connector/pgwal/source/PgwalSourceFactory.java +++ b/chunjun-connectors/chunjun-connector-pgwal/src/main/java/com/dtstack/chunjun/connector/pgwal/source/PgwalSourceFactory.java @@ -6,15 +6,16 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package com.dtstack.chunjun.connector.pgwal.source; import com.dtstack.chunjun.conf.SyncConf; @@ -34,7 +35,6 @@ import org.apache.flink.table.data.RowData; import org.apache.flink.table.types.logical.RowType; -/** */ public class PgwalSourceFactory extends SourceFactory { private final PGWalConf conf; diff --git a/chunjun-connectors/chunjun-connector-postgresql/pom.xml b/chunjun-connectors/chunjun-connector-postgresql/pom.xml index f7d13fd4b6..9f771cb7cd 100644 --- a/chunjun-connectors/chunjun-connector-postgresql/pom.xml +++ b/chunjun-connectors/chunjun-connector-postgresql/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/converter/PostgresqlColumnConverter.java b/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/converter/PostgresqlColumnConverter.java index c19b6c9297..21cb848e05 100644 --- a/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/converter/PostgresqlColumnConverter.java +++ b/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/converter/PostgresqlColumnConverter.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /* * * * diff --git a/chunjun-connectors/chunjun-connector-rabbitmq/pom.xml b/chunjun-connectors/chunjun-connector-rabbitmq/pom.xml index 0bf96d5b4c..82c9f72845 100644 --- a/chunjun-connectors/chunjun-connector-rabbitmq/pom.xml +++ b/chunjun-connectors/chunjun-connector-rabbitmq/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-redis/pom.xml b/chunjun-connectors/chunjun-connector-redis/pom.xml index b068ed4e38..c6238e0dc3 100644 --- a/chunjun-connectors/chunjun-connector-redis/pom.xml +++ b/chunjun-connectors/chunjun-connector-redis/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-rocketmq/pom.xml b/chunjun-connectors/chunjun-connector-rocketmq/pom.xml index 00e531a4ce..3ee06fde1a 100644 --- a/chunjun-connectors/chunjun-connector-rocketmq/pom.xml +++ b/chunjun-connectors/chunjun-connector-rocketmq/pom.xml @@ -1,4 +1,23 @@ + + + diff --git a/chunjun-connectors/chunjun-connector-saphana/pom.xml b/chunjun-connectors/chunjun-connector-saphana/pom.xml index 5088683787..65067e2574 100644 --- a/chunjun-connectors/chunjun-connector-saphana/pom.xml +++ b/chunjun-connectors/chunjun-connector-saphana/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-socket/pom.xml b/chunjun-connectors/chunjun-connector-socket/pom.xml index cd65c93719..86db71798f 100644 --- a/chunjun-connectors/chunjun-connector-socket/pom.xml +++ b/chunjun-connectors/chunjun-connector-socket/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-socket/src/main/java/com/dtstack/chunjun/connector/socket/source/SocketSourceFactory.java b/chunjun-connectors/chunjun-connector-socket/src/main/java/com/dtstack/chunjun/connector/socket/source/SocketSourceFactory.java index de4383d803..7f5dbb5ef5 100644 --- a/chunjun-connectors/chunjun-connector-socket/src/main/java/com/dtstack/chunjun/connector/socket/source/SocketSourceFactory.java +++ b/chunjun-connectors/chunjun-connector-socket/src/main/java/com/dtstack/chunjun/connector/socket/source/SocketSourceFactory.java @@ -6,15 +6,16 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package com.dtstack.chunjun.connector.socket.source; import com.dtstack.chunjun.conf.SyncConf; @@ -28,11 +29,6 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.data.RowData; -/** - * @company: www.dtstack.com - * @author: toutian - * @create: 2019/7/4 - */ public class SocketSourceFactory extends SourceFactory { private final SocketConfig socketConfig; diff --git a/chunjun-connectors/chunjun-connector-sqlserver/pom.xml b/chunjun-connectors/chunjun-connector-sqlserver/pom.xml index 8c9536e0fa..eb58c32a43 100644 --- a/chunjun-connectors/chunjun-connector-sqlserver/pom.xml +++ b/chunjun-connectors/chunjun-connector-sqlserver/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-sqlservercdc/pom.xml b/chunjun-connectors/chunjun-connector-sqlservercdc/pom.xml index 6aa4423463..c0afd52607 100644 --- a/chunjun-connectors/chunjun-connector-sqlservercdc/pom.xml +++ b/chunjun-connectors/chunjun-connector-sqlservercdc/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-sqlservercdc/src/main/java/com/dtstack/chunjun/connector/sqlservercdc/source/SqlservercdcSourceFactory.java b/chunjun-connectors/chunjun-connector-sqlservercdc/src/main/java/com/dtstack/chunjun/connector/sqlservercdc/source/SqlservercdcSourceFactory.java index 21dcbe823c..9b8882889a 100644 --- a/chunjun-connectors/chunjun-connector-sqlservercdc/src/main/java/com/dtstack/chunjun/connector/sqlservercdc/source/SqlservercdcSourceFactory.java +++ b/chunjun-connectors/chunjun-connector-sqlservercdc/src/main/java/com/dtstack/chunjun/connector/sqlservercdc/source/SqlservercdcSourceFactory.java @@ -6,15 +6,16 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package com.dtstack.chunjun.connector.sqlservercdc.source; import com.dtstack.chunjun.conf.SyncConf; @@ -35,11 +36,6 @@ import org.apache.flink.table.data.RowData; import org.apache.flink.table.types.logical.RowType; -/** - * @company: www.dtstack.com - * @author: toutian - * @create: 2019/7/4 - */ @SuppressWarnings("all") public class SqlservercdcSourceFactory extends SourceFactory { diff --git a/chunjun-connectors/chunjun-connector-starrocks/pom.xml b/chunjun-connectors/chunjun-connector-starrocks/pom.xml index bb0a3b595d..d0fbcc5156 100644 --- a/chunjun-connectors/chunjun-connector-starrocks/pom.xml +++ b/chunjun-connectors/chunjun-connector-starrocks/pom.xml @@ -1,4 +1,23 @@ + + chunjun-connectors diff --git a/chunjun-connectors/chunjun-connector-stream/pom.xml b/chunjun-connectors/chunjun-connector-stream/pom.xml index bb1274b0dd..0d0313f4d6 100644 --- a/chunjun-connectors/chunjun-connector-stream/pom.xml +++ b/chunjun-connectors/chunjun-connector-stream/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-sybase/pom.xml b/chunjun-connectors/chunjun-connector-sybase/pom.xml index d05cbfbf06..ec6995bb95 100644 --- a/chunjun-connectors/chunjun-connector-sybase/pom.xml +++ b/chunjun-connectors/chunjun-connector-sybase/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/conf/Vertica11LookupConf.java b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/conf/Vertica11LookupConf.java index a3fc0829f7..7a69c07f88 100644 --- a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/conf/Vertica11LookupConf.java +++ b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/conf/Vertica11LookupConf.java @@ -1,21 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.connector.vertica11.conf; diff --git a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/converter/Vertica11ColumnConverter.java b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/converter/Vertica11ColumnConverter.java index e72e5a7e7f..971fc73c61 100644 --- a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/converter/Vertica11ColumnConverter.java +++ b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/converter/Vertica11ColumnConverter.java @@ -1,21 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.connector.vertica11.converter; diff --git a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/converter/Vertica11RawTypeConverter.java b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/converter/Vertica11RawTypeConverter.java index 463bc83948..5bf19372cc 100644 --- a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/converter/Vertica11RawTypeConverter.java +++ b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/converter/Vertica11RawTypeConverter.java @@ -1,21 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.connector.vertica11.converter; diff --git a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/converter/Vertica11RowConverter.java b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/converter/Vertica11RowConverter.java index 7e308caf3a..1c8773fd8c 100644 --- a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/converter/Vertica11RowConverter.java +++ b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/converter/Vertica11RowConverter.java @@ -1,21 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.connector.vertica11.converter; diff --git a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/dialect/Vertica11Dialect.java b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/dialect/Vertica11Dialect.java index 7ef49ffa72..7d9cf16ef7 100644 --- a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/dialect/Vertica11Dialect.java +++ b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/dialect/Vertica11Dialect.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.dtstack.chunjun.connector.vertica11.dialect; import com.dtstack.chunjun.conf.ChunJunCommonConf; diff --git a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/lookup/Vertica11LruTableFunction.java b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/lookup/Vertica11LruTableFunction.java index 1f104e6664..7719bf8f24 100644 --- a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/lookup/Vertica11LruTableFunction.java +++ b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/lookup/Vertica11LruTableFunction.java @@ -1,21 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.connector.vertica11.lookup; diff --git a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/lookup/options/Vertica11LookupOptions.java b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/lookup/options/Vertica11LookupOptions.java index 9181678e52..66b76ceab8 100644 --- a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/lookup/options/Vertica11LookupOptions.java +++ b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/lookup/options/Vertica11LookupOptions.java @@ -1,21 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.connector.vertica11.lookup.options; diff --git a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/lookup/provider/Vertica11DataSourceProvider.java b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/lookup/provider/Vertica11DataSourceProvider.java index 7c8994dc79..8898978bae 100644 --- a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/lookup/provider/Vertica11DataSourceProvider.java +++ b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/lookup/provider/Vertica11DataSourceProvider.java @@ -1,21 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.connector.vertica11.lookup.provider; diff --git a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/sink/Vertica11OutputFormat.java b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/sink/Vertica11OutputFormat.java index 2bdafbf399..2d0b90a2ba 100644 --- a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/sink/Vertica11OutputFormat.java +++ b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/sink/Vertica11OutputFormat.java @@ -1,21 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.connector.vertica11.sink; diff --git a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/sink/Vertica11SinkFactory.java b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/sink/Vertica11SinkFactory.java index 1fe4ab6aee..1394a94d9a 100644 --- a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/sink/Vertica11SinkFactory.java +++ b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/sink/Vertica11SinkFactory.java @@ -1,21 +1,19 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.dtstack.chunjun.connector.vertica11.sink; diff --git a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/source/Vertica11DynamicTableSource.java b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/source/Vertica11DynamicTableSource.java index d61a05e90d..791e6a6a9e 100644 --- a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/source/Vertica11DynamicTableSource.java +++ b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/source/Vertica11DynamicTableSource.java @@ -1,21 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.connector.vertica11.source; diff --git a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/table/Vertica11DynamicTableFactory.java b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/table/Vertica11DynamicTableFactory.java index 991cea19f4..69fbd52586 100644 --- a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/table/Vertica11DynamicTableFactory.java +++ b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/table/Vertica11DynamicTableFactory.java @@ -1,21 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.connector.vertica11.table; diff --git a/chunjun-connectors/pom.xml b/chunjun-connectors/pom.xml index 6997127d65..06868d2cbc 100644 --- a/chunjun-connectors/pom.xml +++ b/chunjun-connectors/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-core/.gitignore b/chunjun-core/.gitignore index 9803fe0b0e..3d12a3ef6c 100644 --- a/chunjun-core/.gitignore +++ b/chunjun-core/.gitignore @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + target .idea/ /.idea/* diff --git a/chunjun-core/pom.xml b/chunjun-core/pom.xml index e4c090a64a..4a08791f81 100644 --- a/chunjun-core/pom.xml +++ b/chunjun-core/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/QueuesChamberlain.java b/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/QueuesChamberlain.java index 7eb77f56cf..47d23657ba 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/QueuesChamberlain.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/QueuesChamberlain.java @@ -1,21 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.cdc; diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/worker/Chunk.java b/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/worker/Chunk.java index 545a8e2182..2ea07c1912 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/worker/Chunk.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/worker/Chunk.java @@ -1,21 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.cdc.worker; diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/worker/ChunkSplitter.java b/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/worker/ChunkSplitter.java index d15aef5fca..2fdffe2edb 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/worker/ChunkSplitter.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/worker/ChunkSplitter.java @@ -1,21 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.cdc.worker; diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/worker/Worker.java b/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/worker/Worker.java index 0d8d2647d3..4c507f099f 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/worker/Worker.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/worker/Worker.java @@ -1,21 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.cdc.worker; diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/worker/WorkerManager.java b/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/worker/WorkerManager.java index 6a05f1f057..88b9687350 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/worker/WorkerManager.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/worker/WorkerManager.java @@ -1,21 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.cdc.worker; diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/worker/WorkerOverseer.java b/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/worker/WorkerOverseer.java index 7abb0f4947..8e05701cc8 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/worker/WorkerOverseer.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/cdc/worker/WorkerOverseer.java @@ -1,21 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.cdc.worker; diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/decoder/IDecode.java b/chunjun-core/src/main/java/com/dtstack/chunjun/decoder/IDecode.java index e699ae00ba..d63465ae54 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/decoder/IDecode.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/decoder/IDecode.java @@ -6,24 +6,20 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package com.dtstack.chunjun.decoder; import java.util.Map; -/** - * Date: 2019/11/21 Company: www.dtstack.com - * - * @author tudou - */ public interface IDecode { /** diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/decoder/JsonDecoder.java b/chunjun-core/src/main/java/com/dtstack/chunjun/decoder/JsonDecoder.java index 48a06bc793..e9f7cfbe0d 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/decoder/JsonDecoder.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/decoder/JsonDecoder.java @@ -6,15 +6,16 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package com.dtstack.chunjun.decoder; import com.dtstack.chunjun.util.JsonUtil; @@ -26,11 +27,6 @@ import java.util.Collections; import java.util.Map; -/** - * Date: 2019/11/21 Company: www.dtstack.com - * - * @author tudou - */ public class JsonDecoder implements IDecode, Serializable { private static final long serialVersionUID = 1L; diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/decoder/TextDecoder.java b/chunjun-core/src/main/java/com/dtstack/chunjun/decoder/TextDecoder.java index 8f96534966..f1ea56dfc8 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/decoder/TextDecoder.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/decoder/TextDecoder.java @@ -6,26 +6,22 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package com.dtstack.chunjun.decoder; import java.io.Serializable; import java.util.Collections; import java.util.Map; -/** - * Date: 2019/11/21 Company: www.dtstack.com - * - * @author tudou - */ public class TextDecoder implements IDecode, Serializable { private static final long serialVersionUID = 1L; diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/element/column/SqlArrayColumn.java b/chunjun-core/src/main/java/com/dtstack/chunjun/element/column/SqlArrayColumn.java index ab60891d7b..c8d77321cd 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/element/column/SqlArrayColumn.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/element/column/SqlArrayColumn.java @@ -1,23 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * - * * * Licensed to the Apache Software Foundation (ASF) under one - * * * or more contributor license agreements. See the NOTICE file - * * * distributed with this work for additional information - * * * regarding copyright ownership. The ASF licenses this file - * * * to you under the Apache License, Version 2.0 (the - * * * "License"); you may not use this file except in compliance - * * * with the License. You may obtain a copy of the License at - * * * - * * * http://www.apache.org/licenses/LICENSE-2.0 - * * * - * * * Unless required by applicable law or agreed to in writing, software - * * * distributed under the License is distributed on an "AS IS" BASIS, - * * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * * See the License for the specific language governing permissions and - * * * limitations under the License. - * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.element.column; diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/mapping/Mapping.java b/chunjun-core/src/main/java/com/dtstack/chunjun/mapping/Mapping.java index b5af5238a2..fd3d6be9db 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/mapping/Mapping.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/mapping/Mapping.java @@ -1,21 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.mapping; diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/mapping/MappingConf.java b/chunjun-core/src/main/java/com/dtstack/chunjun/mapping/MappingConf.java index bb0de64351..2076d88ca1 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/mapping/MappingConf.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/mapping/MappingConf.java @@ -1,21 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.mapping; diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/mapping/NameMapping.java b/chunjun-core/src/main/java/com/dtstack/chunjun/mapping/NameMapping.java index 09b9bd00c6..c2a6f52939 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/mapping/NameMapping.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/mapping/NameMapping.java @@ -1,21 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.mapping; diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/mapping/NameMappingFlatMap.java b/chunjun-core/src/main/java/com/dtstack/chunjun/mapping/NameMappingFlatMap.java index d35808c9d2..440ae46d05 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/mapping/NameMappingFlatMap.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/mapping/NameMappingFlatMap.java @@ -1,21 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.dtstack.chunjun.mapping; diff --git a/chunjun-core/src/main/resources/core-default.xml b/chunjun-core/src/main/resources/core-default.xml index 74c47b6ccc..772758594b 100644 --- a/chunjun-core/src/main/resources/core-default.xml +++ b/chunjun-core/src/main/resources/core-default.xml @@ -1,23 +1,25 @@ - - + + diff --git a/chunjun-core/src/test/resources/test_3.sql b/chunjun-core/src/test/resources/test_3.sql index 492902864d..dc3f8e346b 100644 --- a/chunjun-core/src/test/resources/test_3.sql +++ b/chunjun-core/src/test/resources/test_3.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + CREATE TABLE MyResult ( diff --git a/chunjun-ddl/chunjun-ddl-base/pom.xml b/chunjun-ddl/chunjun-ddl-base/pom.xml index a6c7bc8eb3..2a8b2a824f 100644 --- a/chunjun-ddl/chunjun-ddl-base/pom.xml +++ b/chunjun-ddl/chunjun-ddl-base/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-ddl/chunjun-ddl-mysql/pom.xml b/chunjun-ddl/chunjun-ddl-mysql/pom.xml index ce875b1fba..f2ec84cb3e 100644 --- a/chunjun-ddl/chunjun-ddl-mysql/pom.xml +++ b/chunjun-ddl/chunjun-ddl-mysql/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-ddl/chunjun-ddl-oracle/pom.xml b/chunjun-ddl/chunjun-ddl-oracle/pom.xml index 49423e18bc..0ae4e9b4b7 100644 --- a/chunjun-ddl/chunjun-ddl-oracle/pom.xml +++ b/chunjun-ddl/chunjun-ddl-oracle/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-ddl/pom.xml b/chunjun-ddl/pom.xml index 5ca3d0587b..b2c4c7c665 100644 --- a/chunjun-ddl/pom.xml +++ b/chunjun-ddl/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-dev/.env b/chunjun-dev/.env index 12a7f78100..a14a2b5c55 100644 --- a/chunjun-dev/.env +++ b/chunjun-dev/.env @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # 设置时区 TZ=Asia/Shanghai # 设置网络模式 diff --git a/chunjun-dev/app/Dockerfile b/chunjun-dev/app/Dockerfile index 75ce6409e7..277f9181ff 100644 --- a/chunjun-dev/app/Dockerfile +++ b/chunjun-dev/app/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM archlinux:base-devel WORKDIR /tmp diff --git a/chunjun-dev/app/bashrc b/chunjun-dev/app/bashrc index 692f8f6319..ed67d1bddd 100644 --- a/chunjun-dev/app/bashrc +++ b/chunjun-dev/app/bashrc @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [ `alias | grep "^ls=" | wc -l` != 0 ] && unalias ls alias gst='git status -sb' alias ga='git add' diff --git a/chunjun-dev/app/clean_maven.sh b/chunjun-dev/app/clean_maven.sh index 20f6107750..ebdcc10c26 100644 --- a/chunjun-dev/app/clean_maven.sh +++ b/chunjun-dev/app/clean_maven.sh @@ -1,4 +1,21 @@ #!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + #!/bin/bash REPO_PATH="/root/.m2/repository" diff --git a/chunjun-dev/dm7/Dockerfile b/chunjun-dev/dm7/Dockerfile index d199053b86..01f3694e9f 100644 --- a/chunjun-dev/dm7/Dockerfile +++ b/chunjun-dev/dm7/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM liaobaikai/dm7:dev-20210709 LABEL maintainer="dtstack" diff --git a/chunjun-dev/dm8/Dockerfile b/chunjun-dev/dm8/Dockerfile index 2ee2a8c689..09fb9dbb06 100644 --- a/chunjun-dev/dm8/Dockerfile +++ b/chunjun-dev/dm8/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM liaobaikai/dm8:dev-20210813 LABEL maintainer="dtstack" diff --git a/chunjun-dev/docker-compose.yml b/chunjun-dev/docker-compose.yml index 1a591a8690..5cea84ab04 100644 --- a/chunjun-dev/docker-compose.yml +++ b/chunjun-dev/docker-compose.yml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + version: '3.5' # 网络配置 diff --git a/chunjun-dev/kafka2.8/Dockerfile b/chunjun-dev/kafka2.8/Dockerfile index ea2b4028b6..81b2a4dbe7 100644 --- a/chunjun-dev/kafka2.8/Dockerfile +++ b/chunjun-dev/kafka2.8/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM wurstmeister/kafka:2.13-2.8.1 LABEL maintainer="dtstack" diff --git a/chunjun-dev/kudu/Dockerfile b/chunjun-dev/kudu/Dockerfile index 18e93cca37..4c669434bc 100644 --- a/chunjun-dev/kudu/Dockerfile +++ b/chunjun-dev/kudu/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM apache/kudu:latest LABEL maintainer="dtstack" diff --git a/chunjun-dev/mysql5.7/Dockerfile b/chunjun-dev/mysql5.7/Dockerfile index ee7027e468..ec8abf414c 100644 --- a/chunjun-dev/mysql5.7/Dockerfile +++ b/chunjun-dev/mysql5.7/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM mysql:5.7 LABEL maintainer="dtstack" diff --git a/chunjun-dev/mysql8/Dockerfile b/chunjun-dev/mysql8/Dockerfile index 34206a60ab..538abc8312 100644 --- a/chunjun-dev/mysql8/Dockerfile +++ b/chunjun-dev/mysql8/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM mysql:8.0 LABEL maintainer="dtstack" diff --git a/chunjun-dev/redis5/Dockerfile b/chunjun-dev/redis5/Dockerfile index b2d8af167d..dff80681cc 100644 --- a/chunjun-dev/redis5/Dockerfile +++ b/chunjun-dev/redis5/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM redis:5.0 LABEL maintainer="dtstack" diff --git a/chunjun-dev/redis6/Dockerfile b/chunjun-dev/redis6/Dockerfile index 7a4e473989..a061d17df0 100644 --- a/chunjun-dev/redis6/Dockerfile +++ b/chunjun-dev/redis6/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM redis:6.2.5 LABEL maintainer="dtstack" diff --git a/chunjun-dev/zookeeper/Dockerfile b/chunjun-dev/zookeeper/Dockerfile index 8ea1127245..83a2457d33 100644 --- a/chunjun-dev/zookeeper/Dockerfile +++ b/chunjun-dev/zookeeper/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM wurstmeister/zookeeper:latest LABEL maintainer="dtstack" diff --git a/chunjun-dirty/chunjun-dirty-log/pom.xml b/chunjun-dirty/chunjun-dirty-log/pom.xml index 7418f4085f..7a5686ee90 100644 --- a/chunjun-dirty/chunjun-dirty-log/pom.xml +++ b/chunjun-dirty/chunjun-dirty-log/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-dirty/chunjun-dirty-log/src/main/resources/log4j.properties b/chunjun-dirty/chunjun-dirty-log/src/main/resources/log4j.properties index 988d8d2db8..5cdefea2cc 100644 --- a/chunjun-dirty/chunjun-dirty-log/src/main/resources/log4j.properties +++ b/chunjun-dirty/chunjun-dirty-log/src/main/resources/log4j.properties @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ############# # 输出到控制台 ############# diff --git a/chunjun-dirty/chunjun-dirty-log/src/main/resources/logback.xml b/chunjun-dirty/chunjun-dirty-log/src/main/resources/logback.xml index 64afb8d7cc..9319d6a352 100644 --- a/chunjun-dirty/chunjun-dirty-log/src/main/resources/logback.xml +++ b/chunjun-dirty/chunjun-dirty-log/src/main/resources/logback.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-dirty/chunjun-dirty-mysql/pom.xml b/chunjun-dirty/chunjun-dirty-mysql/pom.xml index bbbdb74ea3..18b0628eea 100644 --- a/chunjun-dirty/chunjun-dirty-mysql/pom.xml +++ b/chunjun-dirty/chunjun-dirty-mysql/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-dirty/chunjun-dirty-mysql/src/main/resources/flinkx_dirty_table.sql b/chunjun-dirty/chunjun-dirty-mysql/src/main/resources/flinkx_dirty_table.sql index 554ee799c4..deeaaac8b8 100644 --- a/chunjun-dirty/chunjun-dirty-mysql/src/main/resources/flinkx_dirty_table.sql +++ b/chunjun-dirty/chunjun-dirty-mysql/src/main/resources/flinkx_dirty_table.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + CREATE TABLE IF NOT EXISTS chunjun_dirty_data ( job_id VARCHAR(32) NOT NULL COMMENT 'Flink Job Id', diff --git a/chunjun-dirty/pom.xml b/chunjun-dirty/pom.xml index fa2f10bb3e..b412c728c1 100644 --- a/chunjun-dirty/pom.xml +++ b/chunjun-dirty/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-docker/docker-build/Dockerfile b/chunjun-docker/docker-build/Dockerfile index 056a29ddaf..80343c90cd 100644 --- a/chunjun-docker/docker-build/Dockerfile +++ b/chunjun-docker/docker-build/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM apache/flink:1.12.7-scala_2.12-java8 COPY chunjun-dist.tar.gz /opt/flink/lib/ RUN mkdir /opt/flink/lib/chunjun-dist | tar -zxvf /opt/flink/lib/chunjun-dist.tar.gz -C /opt/flink/lib/chunjun-dist diff --git a/chunjun-docker/docker-build/docker-entrypoint.sh b/chunjun-docker/docker-build/docker-entrypoint.sh index 91e7653aa3..77c22db6d4 100644 --- a/chunjun-docker/docker-build/docker-entrypoint.sh +++ b/chunjun-docker/docker-build/docker-entrypoint.sh @@ -1,4 +1,21 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FILE="/opt/flink/lib/chunjun-dist/chunjun-examples/json/stream/stream.json" JOBTYPE="sync" diff --git a/chunjun-docker/pom.xml b/chunjun-docker/pom.xml index 03d2e99c9f..3442a55c57 100644 --- a/chunjun-docker/pom.xml +++ b/chunjun-docker/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-e2e/pom.xml b/chunjun-e2e/pom.xml index 709b5c7b82..631f4d5df6 100644 --- a/chunjun-e2e/pom.xml +++ b/chunjun-e2e/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-e2e/src/test/resources/docker/emqx/Dockerfile b/chunjun-e2e/src/test/resources/docker/emqx/Dockerfile index 75f68d24af..f1b88c9135 100644 --- a/chunjun-e2e/src/test/resources/docker/emqx/Dockerfile +++ b/chunjun-e2e/src/test/resources/docker/emqx/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM emqx/emqx LABEL maintainer="www.dtstack.com" diff --git a/chunjun-e2e/src/test/resources/docker/flink/local/Dockerfile b/chunjun-e2e/src/test/resources/docker/flink/local/Dockerfile index 6088e13042..cb521def48 100644 --- a/chunjun-e2e/src/test/resources/docker/flink/local/Dockerfile +++ b/chunjun-e2e/src/test/resources/docker/flink/local/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM ubuntu:20.04 ENV SHELL /bin/bash diff --git a/chunjun-e2e/src/test/resources/docker/flink/standalone/Dockerfile b/chunjun-e2e/src/test/resources/docker/flink/standalone/Dockerfile index 33cca35f28..cd2db8fe95 100644 --- a/chunjun-e2e/src/test/resources/docker/flink/standalone/Dockerfile +++ b/chunjun-e2e/src/test/resources/docker/flink/standalone/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM openjdk:8-jdk-alpine LABEL maintainer="www.dtstack.com" diff --git a/chunjun-e2e/src/test/resources/docker/flink/standalone/entrypoint.sh b/chunjun-e2e/src/test/resources/docker/flink/standalone/entrypoint.sh index a935c0acd8..23eb589d42 100644 --- a/chunjun-e2e/src/test/resources/docker/flink/standalone/entrypoint.sh +++ b/chunjun-e2e/src/test/resources/docker/flink/standalone/entrypoint.sh @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + #/bin/bash cd /opt/flink/bin diff --git a/chunjun-e2e/src/test/resources/docker/ftp/Dockerfile b/chunjun-e2e/src/test/resources/docker/ftp/Dockerfile index 454a8e24a7..67553c2009 100644 --- a/chunjun-e2e/src/test/resources/docker/ftp/Dockerfile +++ b/chunjun-e2e/src/test/resources/docker/ftp/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM ubuntu:16.04 ADD test.csv /root/ diff --git a/chunjun-e2e/src/test/resources/docker/mysql/Mysql5Dockerfile b/chunjun-e2e/src/test/resources/docker/mysql/Mysql5Dockerfile index 2b2d93afef..51830c709b 100644 --- a/chunjun-e2e/src/test/resources/docker/mysql/Mysql5Dockerfile +++ b/chunjun-e2e/src/test/resources/docker/mysql/Mysql5Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM mysql:5.7 LABEL maintainer="www.dtstack.com" diff --git a/chunjun-e2e/src/test/resources/docker/mysql/Mysql8Dockerfile b/chunjun-e2e/src/test/resources/docker/mysql/Mysql8Dockerfile index 7c7b1d44cb..58ba4d36c1 100644 --- a/chunjun-e2e/src/test/resources/docker/mysql/Mysql8Dockerfile +++ b/chunjun-e2e/src/test/resources/docker/mysql/Mysql8Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM mysql:8.0 LABEL maintainer="www.dtstack.com" diff --git a/chunjun-e2e/src/test/resources/docker/mysql/init.sql b/chunjun-e2e/src/test/resources/docker/mysql/init.sql index 1cfe5388bd..c4b66b7f6b 100644 --- a/chunjun-e2e/src/test/resources/docker/mysql/init.sql +++ b/chunjun-e2e/src/test/resources/docker/mysql/init.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + create database `chunjun`; set character_set_client = utf8; diff --git a/chunjun-e2e/src/test/resources/docker/oracle/init.sql b/chunjun-e2e/src/test/resources/docker/oracle/init.sql index 890fa10ad7..f781eff031 100644 --- a/chunjun-e2e/src/test/resources/docker/oracle/init.sql +++ b/chunjun-e2e/src/test/resources/docker/oracle/init.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + create table debezium.ALL_TYPE1_SOURCE ( VARCHAR2_TYPE VARCHAR2(255), diff --git a/chunjun-e2e/src/test/resources/docker/postgre/Dockerfile b/chunjun-e2e/src/test/resources/docker/postgre/Dockerfile index 93f8410ffd..50685f4732 100644 --- a/chunjun-e2e/src/test/resources/docker/postgre/Dockerfile +++ b/chunjun-e2e/src/test/resources/docker/postgre/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM debezium/postgres:9.6 LABEL maintainer="www.dtstack.com" diff --git a/chunjun-e2e/src/test/resources/docker/postgre/init.sql b/chunjun-e2e/src/test/resources/docker/postgre/init.sql index 929bf76175..a66093be25 100644 --- a/chunjun-e2e/src/test/resources/docker/postgre/init.sql +++ b/chunjun-e2e/src/test/resources/docker/postgre/init.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + DROP SCHEMA IF EXISTS inventory CASCADE; CREATE SCHEMA inventory; SET search_path TO inventory; diff --git a/chunjun-formats/chunjun-protobuf/pom.xml b/chunjun-formats/chunjun-protobuf/pom.xml index 733d98bb68..5351a408e4 100644 --- a/chunjun-formats/chunjun-protobuf/pom.xml +++ b/chunjun-formats/chunjun-protobuf/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-formats/pom.xml b/chunjun-formats/pom.xml index ad9548c858..09d7726829 100644 --- a/chunjun-formats/pom.xml +++ b/chunjun-formats/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-local-test/pom.xml b/chunjun-local-test/pom.xml index e815b0d0c6..f21c1fdd4d 100644 --- a/chunjun-local-test/pom.xml +++ b/chunjun-local-test/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-local-test/src/main/resources/log4j.properties b/chunjun-local-test/src/main/resources/log4j.properties index 988d8d2db8..5cdefea2cc 100644 --- a/chunjun-local-test/src/main/resources/log4j.properties +++ b/chunjun-local-test/src/main/resources/log4j.properties @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ############# # 输出到控制台 ############# diff --git a/chunjun-local-test/src/main/resources/logback.xml b/chunjun-local-test/src/main/resources/logback.xml index 64afb8d7cc..9319d6a352 100644 --- a/chunjun-local-test/src/main/resources/logback.xml +++ b/chunjun-local-test/src/main/resources/logback.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-metrics/chunjun-metrics-mysql/pom.xml b/chunjun-metrics/chunjun-metrics-mysql/pom.xml index 503ebd00c5..3a2c2b1a63 100644 --- a/chunjun-metrics/chunjun-metrics-mysql/pom.xml +++ b/chunjun-metrics/chunjun-metrics-mysql/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-metrics/chunjun-metrics-prometheus/pom.xml b/chunjun-metrics/chunjun-metrics-prometheus/pom.xml index b7ee385e52..243c96f4e7 100644 --- a/chunjun-metrics/chunjun-metrics-prometheus/pom.xml +++ b/chunjun-metrics/chunjun-metrics-prometheus/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-metrics/chunjun-metrics-rdb/pom.xml b/chunjun-metrics/chunjun-metrics-rdb/pom.xml index 65409fbecc..5417f49533 100644 --- a/chunjun-metrics/chunjun-metrics-rdb/pom.xml +++ b/chunjun-metrics/chunjun-metrics-rdb/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-metrics/pom.xml b/chunjun-metrics/pom.xml index 4874979bff..c857dc68dd 100644 --- a/chunjun-metrics/pom.xml +++ b/chunjun-metrics/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-restore/chunjun-restore-common/pom.xml b/chunjun-restore/chunjun-restore-common/pom.xml index 1a285ffb83..c530f2874a 100644 --- a/chunjun-restore/chunjun-restore-common/pom.xml +++ b/chunjun-restore/chunjun-restore-common/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-restore/chunjun-restore-mysql/pom.xml b/chunjun-restore/chunjun-restore-mysql/pom.xml index 6ecb7e7ab5..6f3326c60d 100644 --- a/chunjun-restore/chunjun-restore-mysql/pom.xml +++ b/chunjun-restore/chunjun-restore-mysql/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/chunjun-restore/chunjun-restore-mysql/src/main/resources/ddl.sql b/chunjun-restore/chunjun-restore-mysql/src/main/resources/ddl.sql index 0401fcd16a..16a731bab4 100644 --- a/chunjun-restore/chunjun-restore-mysql/src/main/resources/ddl.sql +++ b/chunjun-restore/chunjun-restore-mysql/src/main/resources/ddl.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + -- auto-generated definition create table ddl_change ( diff --git a/chunjun-restore/pom.xml b/chunjun-restore/pom.xml index 188171d391..f385f17f1d 100644 --- a/chunjun-restore/pom.xml +++ b/chunjun-restore/pom.xml @@ -1,4 +1,23 @@ + + diff --git a/pom.xml b/pom.xml index f9caf7f47d..70e15dc059 100644 --- a/pom.xml +++ b/pom.xml @@ -1,4 +1,23 @@ + + From 095b76e8bed5ac082ff7079c9afe8c393bb5a2e8 Mon Sep 17 00:00:00 2001 From: kino <56473938+KinoMin@users.noreply.github.com> Date: Fri, 11 Nov 2022 10:51:15 +0800 Subject: [PATCH 03/41] [hotfix][jdbc-connector] pgsql query metadata exception (#1391) (cherry picked from commit 9fcfde3e778f69be9f1ad1354d48a2ef9d4dabd0) --- .../java/com/dtstack/chunjun/connector/jdbc/util/JdbcUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/util/JdbcUtil.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/util/JdbcUtil.java index caa48b8972..50f52733ee 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/util/JdbcUtil.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/util/JdbcUtil.java @@ -166,7 +166,7 @@ public static Pair, List> getTableMetaData( } querySql = String.format("select * from %s where 1=2", tableInfo); } else { - querySql = String.format("select * from ((%s) custom) where 1=2", querySql); + querySql = String.format("select * from (%s) custom where 1=2", querySql); } Statement statement = dbConn.createStatement(); From 6138927868746cd4f3d0f5287e6ab4be294b78dc Mon Sep 17 00:00:00 2001 From: fangxy-7010 <58970862+fangxy-7010@users.noreply.github.com> Date: Fri, 11 Nov 2022 10:53:00 +0800 Subject: [PATCH 04/41] [bugfix][hana]fix hana special char error (#1390) (cherry picked from commit 964388b177672110045918907492d2b1119a5371) --- .../chunjun/connector/saphana/dialect/SaphanaDialect.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chunjun-connectors/chunjun-connector-saphana/src/main/java/com/dtstack/chunjun/connector/saphana/dialect/SaphanaDialect.java b/chunjun-connectors/chunjun-connector-saphana/src/main/java/com/dtstack/chunjun/connector/saphana/dialect/SaphanaDialect.java index d60f188c74..afad5bee6c 100644 --- a/chunjun-connectors/chunjun-connector-saphana/src/main/java/com/dtstack/chunjun/connector/saphana/dialect/SaphanaDialect.java +++ b/chunjun-connectors/chunjun-connector-saphana/src/main/java/com/dtstack/chunjun/connector/saphana/dialect/SaphanaDialect.java @@ -110,7 +110,7 @@ public Optional getUpsertStatement( @Override public String quoteIdentifier(String identifier) { - return identifier; + return "\"" + identifier + "\""; } @Override From d1363c014ba72c65204335d6f2c69210d4eac291 Mon Sep 17 00:00:00 2001 From: AUB Date: Sat, 12 Nov 2022 14:26:59 +0800 Subject: [PATCH 05/41] [Feat-#1392][core] add parameters in json format (#1393) (cherry picked from commit ecc9b8b517eaa2811dacffdda8a95b646eb953e2) --- .../main/java/com/dtstack/chunjun/Main.java | 2 +- .../com/dtstack/chunjun/options/Options.java | 13 +++++++++++- .../com/dtstack/chunjun/util/JobUtil.java | 20 ++++++++++++++----- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/Main.java b/chunjun-core/src/main/java/com/dtstack/chunjun/Main.java index adf8e02b01..34963b12fc 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/Main.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/Main.java @@ -101,7 +101,7 @@ public static void main(String[] args) throws Exception { Options options = new OptionParser(args).getOptions(); String job = URLDecoder.decode(options.getJob(), StandardCharsets.UTF_8.name()); - String replacedJob = JobUtil.replaceJobParameter(options.getP(), job); + String replacedJob = JobUtil.replaceJobParameter(options.getP(), options.getPj(), job); Properties confProperties = PropertiesUtil.parseConf(options.getConfProp()); StreamExecutionEnvironment env = EnvFactory.createStreamExecutionEnvironment(options); StreamTableEnvironment tEnv = diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/options/Options.java b/chunjun-core/src/main/java/com/dtstack/chunjun/options/Options.java index 831533a430..de82f1bc86 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/options/Options.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/options/Options.java @@ -70,9 +70,12 @@ public class Options { @OptionRequired(description = "env properties") private String confProp = "{}"; - @OptionRequired(description = "json modify") + @OptionRequired(description = "parameters in simple format") private String p = ""; + @OptionRequired(description = "parameters in json format") + private String pj = ""; + @OptionRequired(description = "plugin load mode, by classpath or shipfile") private String pluginLoadMode = "shipfile"; @@ -176,6 +179,14 @@ public void setP(String p) { this.p = p; } + public String getPj() { + return pj; + } + + public void setPj(String pj) { + this.pj = pj; + } + public String getPluginLoadMode() { return pluginLoadMode; } diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/util/JobUtil.java b/chunjun-core/src/main/java/com/dtstack/chunjun/util/JobUtil.java index 1d0b80326e..84cab8b7c7 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/util/JobUtil.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/util/JobUtil.java @@ -20,6 +20,8 @@ import com.dtstack.chunjun.constants.ConstantValue; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.type.TypeReference; + import org.apache.commons.lang3.StringUtils; import java.util.HashMap; @@ -31,15 +33,19 @@ private JobUtil() throws IllegalAccessException { throw new IllegalAccessException(getClass().getName() + " can not be instantiated"); } - public static String replaceJobParameter(String p, String job) { + public static String replaceJobParameter(String p, String pj, String job) { if (StringUtils.isNotBlank(p)) { - Map parameters = CommandTransform(p); - job = JsonValueReplace(job, parameters); + Map parameters = commandSimpleTransform(p); + job = jsonValueReplace(job, parameters); + } + if (StringUtils.isNotBlank(pj)) { + Map parameters = commandJsonTransform(pj); + job = jsonValueReplace(job, parameters); } return job; } - public static String JsonValueReplace(String json, Map parameter) { + public static String jsonValueReplace(String json, Map parameter) { for (String item : parameter.keySet()) { if (json.contains("${" + item + "}")) { json = json.replace("${" + item + "}", parameter.get(item)); @@ -49,7 +55,7 @@ public static String JsonValueReplace(String json, Map parameter } /** 将命令行中的修改命令转化为HashMap保存 */ - public static HashMap CommandTransform(String command) { + public static HashMap commandSimpleTransform(String command) { HashMap parameter = new HashMap<>(); String[] split = StringUtils.split(command, ConstantValue.COMMA_SYMBOL); for (String item : split) { @@ -58,4 +64,8 @@ public static HashMap CommandTransform(String command) { } return parameter; } + + public static HashMap commandJsonTransform(String command) { + return JsonUtil.toObject(command, new TypeReference>() {}); + } } From e4ae16c38beff38b0741174e95e92381392b0f57 Mon Sep 17 00:00:00 2001 From: Tyrantlucifer Date: Mon, 14 Nov 2022 16:08:15 +0800 Subject: [PATCH 06/41] [Improve][Connector][bin-log] Unified exception of MapMakerHelper (#1395) (cherry picked from commit 1248cc23b1b65a47d3eb351c7c705aad9989afe5) --- .../main/java/com/google/common/collect/MapMakerHelper.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/chunjun-connectors/chunjun-connector-binlog/src/main/java/com/google/common/collect/MapMakerHelper.java b/chunjun-connectors/chunjun-connector-binlog/src/main/java/com/google/common/collect/MapMakerHelper.java index 6b872409e6..a3a1fec708 100644 --- a/chunjun-connectors/chunjun-connector-binlog/src/main/java/com/google/common/collect/MapMakerHelper.java +++ b/chunjun-connectors/chunjun-connector-binlog/src/main/java/com/google/common/collect/MapMakerHelper.java @@ -17,6 +17,8 @@ */ package com.google.common.collect; +import com.dtstack.chunjun.throwable.ChunJunRuntimeException; + import com.google.common.base.Function; import java.lang.reflect.Method; @@ -31,7 +33,7 @@ public static MapMaker softValues(MapMaker mapMaker) { method.setAccessible(true); return (MapMaker) method.invoke(mapMaker); } catch (Exception e) { - throw new RuntimeException(e); + throw new ChunJunRuntimeException(e); } } @@ -44,7 +46,7 @@ public static ConcurrentMap makeComputingMap( method.setAccessible(true); return (ConcurrentMap) method.invoke(mapMaker, computingFunction); } catch (Exception e) { - throw new RuntimeException(e); + throw new ChunJunRuntimeException(e); } } } From 6c375dc9352448b4270c3b2c7f4bc0904a84efcd Mon Sep 17 00:00:00 2001 From: zhengchaoken Date: Thu, 17 Nov 2022 09:56:30 +0800 Subject: [PATCH 07/41] [Doc][#1402] fix '\r' problems caused by windows (#1403) (cherry picked from commit 43e889393362b18aa3d90e96839fc0c1ec65f2a4) --- README.md | 5 +++++ README_CH.md | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/README.md b/README.md index ca572bee4d..392b131636 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,11 @@ sh bin/chunjun-local.sh -job $SCRIPT_PATH The parameter of "$SCRIPT_PATH" means 'the path where the task script is located'. After execute, you can perform a task locally. +note: +``` +when you package in windows and run sh in linux , you need to execute command sed -i "s/\r//g" bin/*.sh to fix the '\r' problems. +``` + [Reference video](https://www.bilibili.com/video/BV1mT411g7fJ?spm_id_from=333.999.0.0) ### Standalone diff --git a/README_CH.md b/README_CH.md index 8e1e644a51..377f5de663 100644 --- a/README_CH.md +++ b/README_CH.md @@ -127,6 +127,11 @@ sh bin/chunjun-local.sh -job chunjun-examples/json/stream/stream.json 即可执行一个简单的 **stream -> stream** 同步任务 +注意: +``` +如果你是在windows环境下打包，在linux上运行任务前需要执行 sed -i "s/\r//g" bin/*.sh 命令修复sh脚本中的 '\r' 问题。 +``` + [参考视频](https://www.bilibili.com/video/BV1mT411g7fJ?spm_id_from=333.999.0.0) ### Standalone From bb61c522f4c52f4b537f0630ae703021ba3ee47f Mon Sep 17 00:00:00 2001 From: conghe2402 Date: Fri, 18 Nov 2022 16:44:11 +0800 Subject: [PATCH 08/41] fix issue:hdfs inputformat split columns issues and deal with null value of column correctly (#1404) (cherry picked from commit a452025f307bca101bb558115839ef3a5438aadb) --- .../hdfs/converter/HdfsTextColumnConverter.java | 17 +++++++++++++++++ .../hdfs/source/HdfsTextInputFormat.java | 3 ++- .../listener/LogMinerListener.java | 4 +++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/chunjun-connectors/chunjun-connector-hdfs/src/main/java/com/dtstack/chunjun/connector/hdfs/converter/HdfsTextColumnConverter.java b/chunjun-connectors/chunjun-connector-hdfs/src/main/java/com/dtstack/chunjun/connector/hdfs/converter/HdfsTextColumnConverter.java index fd016203ab..07717c71a4 100644 --- a/chunjun-connectors/chunjun-connector-hdfs/src/main/java/com/dtstack/chunjun/connector/hdfs/converter/HdfsTextColumnConverter.java +++ b/chunjun-connectors/chunjun-connector-hdfs/src/main/java/com/dtstack/chunjun/connector/hdfs/converter/HdfsTextColumnConverter.java @@ -107,6 +107,23 @@ public RowData toInternalLookup(RowData input) { throw new ChunJunRuntimeException("HDFS Connector doesn't support Lookup Table Function."); } + @Override + protected IDeserializationConverter wrapIntoNullableInternalConverter( + IDeserializationConverter IDeserializationConverter) { + return val -> { + if (val == null || "".equals(val)) { + return null; + } else { + try { + return IDeserializationConverter.deserialize(val); + } catch (Exception e) { + LOG.error("value [{}] convent failed ", val); + throw e; + } + } + }; + } + @Override @SuppressWarnings("unchecked") protected ISerializationConverter wrapIntoNullableExternalConverter( diff --git a/chunjun-connectors/chunjun-connector-hdfs/src/main/java/com/dtstack/chunjun/connector/hdfs/source/HdfsTextInputFormat.java b/chunjun-connectors/chunjun-connector-hdfs/src/main/java/com/dtstack/chunjun/connector/hdfs/source/HdfsTextInputFormat.java index 38b3989827..3e0b9b13c3 100644 --- a/chunjun-connectors/chunjun-connector-hdfs/src/main/java/com/dtstack/chunjun/connector/hdfs/source/HdfsTextInputFormat.java +++ b/chunjun-connectors/chunjun-connector-hdfs/src/main/java/com/dtstack/chunjun/connector/hdfs/source/HdfsTextInputFormat.java @@ -123,7 +123,8 @@ public RowData nextRecordInternal(RowData rowData) throws ReadRecordException { ((Text) value).getLength(), hdfsConf.getEncoding()); String[] fields = - StringUtils.splitPreserveAllTokens(line, hdfsConf.getFieldDelimiter()); + StringUtils.splitByWholeSeparatorPreserveAllTokens( + line, hdfsConf.getFieldDelimiter()); List fieldConfList = hdfsConf.getColumn(); GenericRowData genericRowData; diff --git a/chunjun-connectors/chunjun-connector-oraclelogminer/src/main/java/com/dtstack/chunjun/connector/oraclelogminer/listener/LogMinerListener.java b/chunjun-connectors/chunjun-connector-oraclelogminer/src/main/java/com/dtstack/chunjun/connector/oraclelogminer/listener/LogMinerListener.java index 0ce7479cb5..abb73278e2 100644 --- a/chunjun-connectors/chunjun-connector-oraclelogminer/src/main/java/com/dtstack/chunjun/connector/oraclelogminer/listener/LogMinerListener.java +++ b/chunjun-connectors/chunjun-connector-oraclelogminer/src/main/java/com/dtstack/chunjun/connector/oraclelogminer/listener/LogMinerListener.java @@ -524,7 +524,9 @@ private BigInteger getLockTableScn(Connection conn, String tbnWithSchema) { } /* generate create table ddl */ - initialTableStruct(conn); + if (logMinerConf.isInitialTableStructure()) { + initialTableStruct(conn); + } /* release lock */ stmt.execute(SqlUtil.releaseTableLock()); From 1198c92da957d3ab18bcbe9836092d6407a13c96 Mon Sep 17 00:00:00 2001 From: zhengchaoken Date: Fri, 18 Nov 2022 17:05:59 +0800 Subject: [PATCH 09/41] fix BUG #1398,fix bsh,xml-resolver default version cannot access (#1399) * fix BUG #1398,fix bsh,xml-resolver default version cannot access * fix BUG #1400,fix multiple SLF4J bindings & no log logged in standalone,yarn-session,yarn-prejob (cherry picked from commit 3ae4a9f6b850c660e691a91a1c16aaf3319ec57d) --- bin/submit.sh | 4 +- chunjun-assembly/pom.xml | 37 ++++++++++++++++++- .../src/main/assembly/assembly.xml | 6 ++- chunjun-clients/pom.xml | 11 ++++++ .../chunjun-connector-hdfs/pom.xml | 8 +++- .../chunjun-connector-hive/pom.xml | 11 ++++++ chunjun-core/pom.xml | 8 ++-- chunjun-ddl/chunjun-ddl-mysql/pom.xml | 18 ++++++++- chunjun-restore/chunjun-restore-mysql/pom.xml | 8 +++- 9 files changed, 97 insertions(+), 14 deletions(-) diff --git a/bin/submit.sh b/bin/submit.sh index db1de0fa6a..e96d218b3d 100644 --- a/bin/submit.sh +++ b/bin/submit.sh @@ -46,9 +46,9 @@ fi # 1.In yarn-session case, JAR_DIR can not be found # 2.In other cases, JAR_DIR can be found if [ $CHUNJUN_DEPLOY_MODE -eq 1 ]; then - JAR_DIR=$CHUNJUN_HOME/lib/* + JAR_DIR=$CHUNJUN_HOME/lib/chunjun-clients.jar:$CHUNJUN_HOME/lib/* else - JAR_DIR=$CHUNJUN_HOME/../lib/* + JAR_DIR=$CHUNJUN_HOME/../lib/chunjun-clients.jar:$CHUNJUN_HOME/../lib/* fi CLASS_NAME=com.dtstack.chunjun.client.Launcher diff --git a/chunjun-assembly/pom.xml b/chunjun-assembly/pom.xml index d353c3b7d8..08bac891f1 100644 --- a/chunjun-assembly/pom.xml +++ b/chunjun-assembly/pom.xml @@ -33,15 +33,50 @@ pom ChunJun : Assembly - true + + 2.16.0 ${project.parent.basedir} assembly none package + + + + org.apache.logging.log4j + log4j-1.2-api + ${log4j2.version} + provided + + + org.apache.logging.log4j + log4j-api + ${log4j2.version} + provided + + + org.apache.logging.log4j + log4j-core + ${log4j2.version} + provided + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j2.version} + provided + + + slf4j-api + org.slf4j + + + + + diff --git a/chunjun-assembly/src/main/assembly/assembly.xml b/chunjun-assembly/src/main/assembly/assembly.xml index bcdf61912c..686a610651 100644 --- a/chunjun-assembly/src/main/assembly/assembly.xml +++ b/chunjun-assembly/src/main/assembly/assembly.xml @@ -59,8 +59,12 @@ lib true false - package + provided false + + + org.apache.logging.log4j:* + diff --git a/chunjun-clients/pom.xml b/chunjun-clients/pom.xml index 69b3dabb3e..673d450347 100644 --- a/chunjun-clients/pom.xml +++ b/chunjun-clients/pom.xml @@ -233,6 +233,17 @@ shade + + + + org.slf4j:* + org.apache.logging.log4j:* + ch.qos.logback:* + ch.qos.reload4j:* + commons-logging:* + log4j:log4j + + false diff --git a/chunjun-connectors/chunjun-connector-hdfs/pom.xml b/chunjun-connectors/chunjun-connector-hdfs/pom.xml index 9faa33edac..56be739072 100644 --- a/chunjun-connectors/chunjun-connector-hdfs/pom.xml +++ b/chunjun-connectors/chunjun-connector-hdfs/pom.xml @@ -228,10 +228,14 @@ + - org.slf4j:slf4j-api - log4j:log4j + org.slf4j:* + org.apache.logging.log4j:* ch.qos.logback:* + ch.qos.reload4j:* + commons-logging:* + log4j:log4j diff --git a/chunjun-connectors/chunjun-connector-hive/pom.xml b/chunjun-connectors/chunjun-connector-hive/pom.xml index 0e32951296..4e12b5a701 100644 --- a/chunjun-connectors/chunjun-connector-hive/pom.xml +++ b/chunjun-connectors/chunjun-connector-hive/pom.xml @@ -285,6 +285,17 @@ false + + + + org.slf4j:* + org.apache.logging.log4j:* + ch.qos.logback:* + ch.qos.reload4j:* + commons-logging:* + log4j:log4j + + *:* diff --git a/chunjun-core/pom.xml b/chunjun-core/pom.xml index 4a08791f81..bcac974053 100644 --- a/chunjun-core/pom.xml +++ b/chunjun-core/pom.xml @@ -451,10 +451,10 @@ + + org.slf4j:slf4j-api com.google.guava:* com.google.code.gson:* - ch.qos.logback:* - org.slf4j:* org.apache.httpcomponents:* io.prometheus:* org.apache.avro:* @@ -508,10 +508,10 @@ + + org.slf4j:slf4j-api com.google.guava:* com.google.code.gson:* - ch.qos.logback:* - org.slf4j:* org.apache.httpcomponents:* io.prometheus:* org.apache.avro:* diff --git a/chunjun-ddl/chunjun-ddl-mysql/pom.xml b/chunjun-ddl/chunjun-ddl-mysql/pom.xml index f2ec84cb3e..3e099a2d32 100644 --- a/chunjun-ddl/chunjun-ddl-mysql/pom.xml +++ b/chunjun-ddl/chunjun-ddl-mysql/pom.xml @@ -130,6 +130,16 @@ freemarker 2.3.28 + + org.beanshell + bsh + 2.0b5 + + + xml-resolver + xml-resolver + 1.2 + @@ -184,9 +194,13 @@ - org.slf4j:slf4j-api - log4j:log4j + + org.slf4j:* + org.apache.logging.log4j:* ch.qos.logback:* + ch.qos.reload4j:* + commons-logging:* + log4j:log4j diff --git a/chunjun-restore/chunjun-restore-mysql/pom.xml b/chunjun-restore/chunjun-restore-mysql/pom.xml index 6f3326c60d..b6efd31fc9 100644 --- a/chunjun-restore/chunjun-restore-mysql/pom.xml +++ b/chunjun-restore/chunjun-restore-mysql/pom.xml @@ -83,9 +83,13 @@ - org.slf4j:slf4j-api - log4j:log4j + + org.slf4j:* + org.apache.logging.log4j:* ch.qos.logback:* + ch.qos.reload4j:* + commons-logging:* + log4j:log4j From d863ff63d8280ccf1783730d6d7f0ef72068eb52 Mon Sep 17 00:00:00 2001 From: fangxy-7010 <58970862+fangxy-7010@users.noreply.github.com> Date: Sun, 20 Nov 2022 14:36:20 +0800 Subject: [PATCH 10/41] [bugfix][hbase]fix hbase date type (#1406) Co-authored-by: linchen (cherry picked from commit 703f79c2166018604b865ea2f8ab91406170da1e) --- .../connector/hbase/converter/HBaseColumnConverter.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/converter/HBaseColumnConverter.java b/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/converter/HBaseColumnConverter.java index 80e1d884a2..646d9a6bc0 100644 --- a/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/converter/HBaseColumnConverter.java +++ b/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/converter/HBaseColumnConverter.java @@ -430,6 +430,12 @@ protected ISerializationConverter createExternalConverter(LogicalType ((Put) output).addColumn(qualifier[0], qualifier[1], bytes); }; case DATE: + return (rowData, pos, output) -> { + String value = ((ColumnRowData) rowData).getField(pos).asSqlDate().toString(); + byte[] bytes = Bytes.toBytes(value); + byte[][] qualifier = familyAndQualifier[pos]; + ((Put) output).addColumn(qualifier[0], qualifier[1], bytes); + }; case INTERVAL_YEAR_MONTH: case INTEGER: return (rowData, pos, output) -> { From 0ac81e1b404185cd2667811713e7936c87452767 Mon Sep 17 00:00:00 2001 From: conghe2402 Date: Tue, 22 Nov 2022 11:57:17 +0800 Subject: [PATCH 11/41] [hotfix][hdfs]when writing to hdfs, null value is set to '\N' (#1408) [hotfix][hdfs]when writing to hdfs, null value is set to '\N' for hive specification so that hdfs connector can read from the output (cherry picked from commit a924b98866be0cb985b80d6b77de18aabdb5d093) --- .../connector/hdfs/converter/HdfsTextColumnConverter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chunjun-connectors/chunjun-connector-hdfs/src/main/java/com/dtstack/chunjun/connector/hdfs/converter/HdfsTextColumnConverter.java b/chunjun-connectors/chunjun-connector-hdfs/src/main/java/com/dtstack/chunjun/connector/hdfs/converter/HdfsTextColumnConverter.java index 07717c71a4..121bd87f26 100644 --- a/chunjun-connectors/chunjun-connector-hdfs/src/main/java/com/dtstack/chunjun/connector/hdfs/converter/HdfsTextColumnConverter.java +++ b/chunjun-connectors/chunjun-connector-hdfs/src/main/java/com/dtstack/chunjun/connector/hdfs/converter/HdfsTextColumnConverter.java @@ -130,7 +130,7 @@ protected ISerializationConverter wrapIntoNullableExternalConverter( ISerializationConverter serializationConverter, String type) { return (rowData, index, data) -> { if (rowData == null || rowData.isNullAt(index)) { - data[index] = null; + data[index] = "\\N"; } else { serializationConverter.serialize(rowData, index, data); } From 89e2d2b1d25bb621f7cbf0891d13ea33aab31392 Mon Sep 17 00:00:00 2001 From: superPiKaChu <1623857502@qq.com> Date: Thu, 1 Dec 2022 09:52:32 +0800 Subject: [PATCH 12/41] [doc-#1415] update doc & starrocks batchsize (#1417) (cherry picked from commit 41cb73a0a6bfac38cfaa3b0228caa19ee754c05d) --- .../starrocks/sink/StarRocksSinkFactory.java | 2 +- .../hbase/hbase-sink.md" | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/chunjun-connectors/chunjun-connector-starrocks/src/main/java/com/dtstack/chunjun/connector/starrocks/sink/StarRocksSinkFactory.java b/chunjun-connectors/chunjun-connector-starrocks/src/main/java/com/dtstack/chunjun/connector/starrocks/sink/StarRocksSinkFactory.java index 615394d2df..a2a7e25a1e 100644 --- a/chunjun-connectors/chunjun-connector-starrocks/src/main/java/com/dtstack/chunjun/connector/starrocks/sink/StarRocksSinkFactory.java +++ b/chunjun-connectors/chunjun-connector-starrocks/src/main/java/com/dtstack/chunjun/connector/starrocks/sink/StarRocksSinkFactory.java @@ -50,7 +50,7 @@ public StarRocksSinkFactory(SyncConf syncConf) { JsonUtil.toObject( JsonUtil.toJson(syncConf.getWriter().getParameter()), StarRocksConf.class); - int batchSize = syncConf.getWriter().getIntVal("batchSize", 10240); + int batchSize = syncConf.getWriter().getIntVal("batchSize", 1024); starRocksConf.setBatchSize(batchSize); super.initCommonConf(starRocksConf); } diff --git "a/docs_zh/ChunJun\350\277\236\346\216\245\345\231\250/hbase/hbase-sink.md" "b/docs_zh/ChunJun\350\277\236\346\216\245\345\231\250/hbase/hbase-sink.md" index 18eb33e4f5..c702364750 100644 --- "a/docs_zh/ChunJun\350\277\236\346\216\245\345\231\250/hbase/hbase-sink.md" +++ "b/docs_zh/ChunJun\350\277\236\346\216\245\345\231\250/hbase/hbase-sink.md" @@ -27,6 +27,17 @@ hbase1.4 - 默认值：无 +- **column** + + - 描述：要写入的hbase字段。 + - name：指定写入的hbase列，必须为 列族:列名 的格式； + - type：指定源数据的类型，format指定日期类型的格式 + - 必选：是 + - 字段类型：List + - 默认值：无 + - 注意：为了支持数据中某一列当做row_key这种情况，设置字符串rowkey为关键字，如果某一列name为rowkey，那么该列将不会写入 + + - **encoding** - 描述：编码 - 必选：否 @@ -58,6 +69,13 @@ hbase1.4
+- **batchSize** + + - 描述：一次性批量提交的记录数大小，该值可以极大减少ChunJun与数据库的网络交互次数，并提升整体吞吐量。但是该值设置过大可能会造成ChunJun运行进程OOM情况 + - 必选：否 + - 参数类型：int + - 默认值：1 + - **rowkeyExpress** - 描述： 用于构造rowkey的描述信息，采用字符串格式，形式如下 From 9b7a63706d1738f54c34d8097ff3ecee39b9c2e5 Mon Sep 17 00:00:00 2001 From: superPiKaChu <1623857502@qq.com> Date: Sun, 4 Dec 2022 09:10:15 +0800 Subject: [PATCH 13/41] [Improve][Clinet-#1426] delete tmp flink-distribute-cache-xxx dir after job is done (#1427) (cherry picked from commit c664cc2d3baa245077fe0df9c55fbeb45bb06add) --- .../environment/MyLocalStreamEnvironment.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/environment/MyLocalStreamEnvironment.java b/chunjun-core/src/main/java/com/dtstack/chunjun/environment/MyLocalStreamEnvironment.java index f55b541eca..80cc0cef23 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/environment/MyLocalStreamEnvironment.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/environment/MyLocalStreamEnvironment.java @@ -21,6 +21,7 @@ import org.apache.flink.annotation.Public; import org.apache.flink.api.common.InvalidProgramException; import org.apache.flink.api.common.JobExecutionResult; +import org.apache.flink.api.common.JobID; import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.DeploymentOptions; @@ -33,11 +34,14 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.graph.StreamGraph; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; +import java.io.File; +import java.io.IOException; import java.net.URL; import java.util.Collections; import java.util.List; @@ -113,6 +117,22 @@ private static Configuration validateAndGetConfiguration(final Configuration con return effectiveConfiguration; } + private void clearFlinkLocalDsitributeCache(JobGraph jobGraph) throws IOException { + File osTmpDir = new File(System.getProperty("java.io.tmpdir")); + JobID jobID = jobGraph.getJobID(); + File[] flinkTmpDirs = + osTmpDir.listFiles( + (dir, name) -> name.startsWith("flink-distributed-cache-" + jobID)); + + if (flinkTmpDirs != null && flinkTmpDirs.length > 0) { + for (File flinkTmpDir : flinkTmpDirs) { + if (flinkTmpDir.exists() && flinkTmpDir.isDirectory()) { + FileUtils.deleteDirectory(flinkTmpDir); + } + } + } + } + @Override protected Configuration getConfiguration() { return configuration; @@ -175,6 +195,7 @@ public JobExecutionResult execute(String jobName) throws Exception { } finally { transformations.clear(); miniCluster.close(); + clearFlinkLocalDsitributeCache(jobGraph); } } } From f3372f8d007443bda94bd75f2343b408252989d9 Mon Sep 17 00:00:00 2001 From: conghe2402 Date: Sun, 4 Dec 2022 09:24:30 +0800 Subject: [PATCH 14/41] [Improve][Hive] Add Hive table check and correct function (#1425) (cherry picked from commit 1c9c789e2b7c61980139dce00a8032a0b6e5fd7b) --- .../hive/util/PathConverterUtil.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/chunjun-connectors/chunjun-connector-hive/src/main/java/com/dtstack/chunjun/connector/hive/util/PathConverterUtil.java b/chunjun-connectors/chunjun-connector-hive/src/main/java/com/dtstack/chunjun/connector/hive/util/PathConverterUtil.java index fa453bed05..9a123f27c0 100644 --- a/chunjun-connectors/chunjun-connector-hive/src/main/java/com/dtstack/chunjun/connector/hive/util/PathConverterUtil.java +++ b/chunjun-connectors/chunjun-connector-hive/src/main/java/com/dtstack/chunjun/connector/hive/util/PathConverterUtil.java @@ -19,7 +19,9 @@ import com.dtstack.chunjun.element.AbstractBaseColumn; import com.dtstack.chunjun.element.ColumnRowData; +import com.dtstack.chunjun.throwable.ChunJunRuntimeException; +import org.datanucleus.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +39,8 @@ public class PathConverterUtil { private static final Pattern pat1 = Pattern.compile("\\$\\{.*?\\}"); private static final String KEY_TABLE = "table"; + private static final String HIVE_TABLE_PATTERN = "[A-Za-z0-9][A-Za-z0-9_]*"; + /** * @param path * @return @@ -59,6 +63,7 @@ public static String regexByRules( // .在sql中会视为db.table的分隔符，需要单独过滤特殊字符 '.' path = path.replace(pkey, ruleValue).replace(".", "_"); } + path = tryGetValidHiveTableName(path); } catch (Exception e) { logger.error("parser path rules is fail", e); } @@ -87,10 +92,25 @@ public static String regexByRules( } // .在sql中会视为db.table的分隔符，需要单独过滤特殊字符 '.' path = path.replace(pkey, ruleValue).replace(".", "_"); + path = tryGetValidHiveTableName(path); } } catch (Exception e) { logger.error("parser path rules is fail", e); } return path; } + + private static String tryGetValidHiveTableName(String path) { + // 各种来源的表名 schema.tablename 并不会与 hive table name 规范兼容，这里需要适配 + Pattern hiveTableNamePattern = Pattern.compile(HIVE_TABLE_PATTERN); + Matcher hiveTableMatcher = hiveTableNamePattern.matcher(path); + StringBuilder validTableName = new StringBuilder(); + while (hiveTableMatcher.find()) { + validTableName.append(hiveTableMatcher.group()); + } + if (StringUtils.isEmpty(validTableName.toString())) { + throw new ChunJunRuntimeException("hive table name does not match"); + } + return validTableName.toString(); + } } From 17de82a6b3d41c237768a0f432e8b941440defb9 Mon Sep 17 00:00:00 2001 From: congheGOGO Date: Mon, 5 Dec 2022 13:54:22 +0800 Subject: [PATCH 15/41] [Improve][Connector][Hive] add hive table check and correct function (#1429) (cherry picked from commit 91c3bd48e0a04c2b8132e8e98f3e1b50c3298ae4) --- .../dtstack/chunjun/connector/hive/util/PathConverterUtil.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/chunjun-connectors/chunjun-connector-hive/src/main/java/com/dtstack/chunjun/connector/hive/util/PathConverterUtil.java b/chunjun-connectors/chunjun-connector-hive/src/main/java/com/dtstack/chunjun/connector/hive/util/PathConverterUtil.java index 9a123f27c0..ed1c19e8ac 100644 --- a/chunjun-connectors/chunjun-connector-hive/src/main/java/com/dtstack/chunjun/connector/hive/util/PathConverterUtil.java +++ b/chunjun-connectors/chunjun-connector-hive/src/main/java/com/dtstack/chunjun/connector/hive/util/PathConverterUtil.java @@ -92,8 +92,9 @@ public static String regexByRules( } // .在sql中会视为db.table的分隔符，需要单独过滤特殊字符 '.' path = path.replace(pkey, ruleValue).replace(".", "_"); - path = tryGetValidHiveTableName(path); } + + path = tryGetValidHiveTableName(path); } catch (Exception e) { logger.error("parser path rules is fail", e); } From bff63f9732fef6ff4bccb96f92294acb9fcf1f43 Mon Sep 17 00:00:00 2001 From: kyotom <37698246+kyo-tom@users.noreply.github.com> Date: Wed, 7 Dec 2022 11:23:28 +0800 Subject: [PATCH 16/41] [Improve][Client] Chunjun client support submit task to hadoop with kerberos(#1435) (cherry picked from commit 0784968b7d15bdd075adcab940f05d897d29534d) --- .../yarn/YarnSessionClusterClientHelper.java | 110 +++++++++++------- 1 file changed, 67 insertions(+), 43 deletions(-) diff --git a/chunjun-clients/src/main/java/com/dtstack/chunjun/client/yarn/YarnSessionClusterClientHelper.java b/chunjun-clients/src/main/java/com/dtstack/chunjun/client/yarn/YarnSessionClusterClientHelper.java index 88f245d92e..dfbaa96fca 100644 --- a/chunjun-clients/src/main/java/com/dtstack/chunjun/client/yarn/YarnSessionClusterClientHelper.java +++ b/chunjun-clients/src/main/java/com/dtstack/chunjun/client/yarn/YarnSessionClusterClientHelper.java @@ -29,6 +29,8 @@ import org.apache.flink.core.fs.FileSystem; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobmanager.HighAvailabilityMode; +import org.apache.flink.runtime.security.SecurityConfiguration; +import org.apache.flink.runtime.security.SecurityUtils; import org.apache.flink.yarn.YarnClientYarnClusterInformationRetriever; import org.apache.flink.yarn.YarnClusterDescriptor; @@ -46,6 +48,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.Callable; import static org.apache.flink.yarn.configuration.YarnConfigOptions.APPLICATION_ID; import static org.apache.flink.yarn.configuration.YarnConfigOptions.APPLICATION_QUEUE; @@ -68,50 +71,71 @@ public ClusterClient submit(JobDeployer jobDeployer) throws Exception { String yarnConfDir = launcherOptions.getHadoopConfDir(); try { - FileSystem.initialize(flinkConfig); - YarnConfiguration yarnConf = YarnConfLoader.getYarnConf(yarnConfDir); - try (YarnClient yarnClient = YarnClient.createYarnClient()) { - yarnClient.init(yarnConf); - yarnClient.start(); - ApplicationId applicationId; - - if (StringUtils.isEmpty(flinkConfig.get(APPLICATION_ID))) { - applicationId = getAppIdFromYarn(yarnClient, flinkConfig); - if (applicationId == null || StringUtils.isEmpty(applicationId.toString())) { - throw new RuntimeException("No flink session found on yarn cluster."); - } - } else { - applicationId = ConverterUtils.toApplicationId(flinkConfig.get(APPLICATION_ID)); - } - - HighAvailabilityMode highAvailabilityMode = - HighAvailabilityMode.fromConfig(flinkConfig); - if (highAvailabilityMode.equals(HighAvailabilityMode.ZOOKEEPER) - && applicationId != null) { - flinkConfig.setString( - HighAvailabilityOptions.HA_CLUSTER_ID, applicationId.toString()); - } - try (YarnClusterDescriptor yarnClusterDescriptor = - new YarnClusterDescriptor( - flinkConfig, - yarnConf, - yarnClient, - YarnClientYarnClusterInformationRetriever.create(yarnClient), - true)) { - ClusterClient clusterClient = - yarnClusterDescriptor.retrieve(applicationId).getClusterClient(); - JobGraph jobGraph = - JobGraphUtil.buildJobGraph( - launcherOptions, programArgs.toArray(new String[0])); - jobGraph.getClasspaths().clear(); - jobGraph.getUserJars().clear(); - jobGraph.getUserArtifacts().clear(); - JobID jobID = (JobID) clusterClient.submitJob(jobGraph).get(); - LOG.info("submit job successfully, jobID = {}", jobID); - return clusterClient; - } - } + + SecurityUtils.install(new SecurityConfiguration(flinkConfig)); + return SecurityUtils.getInstalledContext() + .runSecured( + new Callable() { + @Override + public ClusterClient call() throws Exception { + FileSystem.initialize(flinkConfig); + try (YarnClient yarnClient = YarnClient.createYarnClient()) { + yarnClient.init(yarnConf); + yarnClient.start(); + ApplicationId applicationId; + + if (StringUtils.isEmpty(flinkConfig.get(APPLICATION_ID))) { + applicationId = + getAppIdFromYarn(yarnClient, flinkConfig); + if (applicationId == null + || StringUtils.isEmpty( + applicationId.toString())) { + throw new RuntimeException( + "No flink session found on yarn cluster."); + } + } else { + applicationId = + ConverterUtils.toApplicationId( + flinkConfig.get(APPLICATION_ID)); + } + + HighAvailabilityMode highAvailabilityMode = + HighAvailabilityMode.fromConfig(flinkConfig); + if (highAvailabilityMode.equals( + HighAvailabilityMode.ZOOKEEPER) + && applicationId != null) { + flinkConfig.setString( + HighAvailabilityOptions.HA_CLUSTER_ID, + applicationId.toString()); + } + try (YarnClusterDescriptor yarnClusterDescriptor = + new YarnClusterDescriptor( + flinkConfig, + yarnConf, + yarnClient, + YarnClientYarnClusterInformationRetriever + .create(yarnClient), + true)) { + ClusterClient clusterClient = + yarnClusterDescriptor + .retrieve(applicationId) + .getClusterClient(); + JobGraph jobGraph = + JobGraphUtil.buildJobGraph( + launcherOptions, + programArgs.toArray(new String[0])); + jobGraph.getClasspaths().clear(); + jobGraph.getUserJars().clear(); + jobGraph.getUserArtifacts().clear(); + JobID jobID = + (JobID) clusterClient.submitJob(jobGraph).get(); + LOG.info("submit job successfully, jobID = {}", jobID); + return clusterClient; + } + } + } + }); } catch (Exception e) { throw new RuntimeException(e); } From 0df6ff03840898a6b77f00a793b0f93d886076a0 Mon Sep 17 00:00:00 2001 From: Jia He <491311184@qq.com> Date: Wed, 7 Dec 2022 17:26:20 +0800 Subject: [PATCH 17/41] [Hotfix][CDC] Fix:mysql-cdc can't run in sql mode (#1436) com.ververica.cdc.connectors.mysql.table.MySqlTableSourceFactory not merge in META-INF/services/org.apache.flink.table.factories.Factory when maven package Signed-off-by: Jia He <491311184@qq.com> Signed-off-by: Jia He <491311184@qq.com> (cherry picked from commit d5b464374bd3d24e10296afdfe55626d361cc362) --- chunjun-connectors/chunjun-connector-mysqlcdc/pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/chunjun-connectors/chunjun-connector-mysqlcdc/pom.xml b/chunjun-connectors/chunjun-connector-mysqlcdc/pom.xml index 77109c0ebc..29ef4d0a6c 100644 --- a/chunjun-connectors/chunjun-connector-mysqlcdc/pom.xml +++ b/chunjun-connectors/chunjun-connector-mysqlcdc/pom.xml @@ -91,6 +91,9 @@ + + + From e233c61bbecfac32fcad08a3824edcc490c1c3f2 Mon Sep 17 00:00:00 2001 From: superPiKaChu <1623857502@qq.com> Date: Fri, 9 Dec 2022 09:19:07 +0800 Subject: [PATCH 18/41] [BugFix-#1334][hdfs-connector] fix orc timestamp/date column can't read (#1383) (cherry picked from commit 6bb1615683822f3e68e4fd14065495ae791d713b) --- .../hdfs/converter/HdfsOrcColumnConverter.java | 13 ++++++++++--- .../chunjun/connector/hdfs/util/HdfsUtil.java | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/chunjun-connectors/chunjun-connector-hdfs/src/main/java/com/dtstack/chunjun/connector/hdfs/converter/HdfsOrcColumnConverter.java b/chunjun-connectors/chunjun-connector-hdfs/src/main/java/com/dtstack/chunjun/connector/hdfs/converter/HdfsOrcColumnConverter.java index a4df99af9a..1eb139e6f7 100644 --- a/chunjun-connectors/chunjun-connector-hdfs/src/main/java/com/dtstack/chunjun/connector/hdfs/converter/HdfsOrcColumnConverter.java +++ b/chunjun-connectors/chunjun-connector-hdfs/src/main/java/com/dtstack/chunjun/connector/hdfs/converter/HdfsOrcColumnConverter.java @@ -222,11 +222,18 @@ protected ISerializationConverter createExternalConverter(String type) case "CHAR": return (rowData, index, data) -> data[index] = rowData.getString(index).toString(); case "TIMESTAMP": - return (rowData, index, data) -> - data[index] = rowData.getTimestamp(index, 6).toTimestamp(); + return (rowData, index, data) -> { + Timestamp ts = rowData.getTimestamp(index, 6).toTimestamp(); + int nanos = ts.getNanos(); + data[index] = + org.apache.hadoop.hive.common.type.Timestamp.ofEpochMilli( + ts.getTime(), nanos); + }; case "DATE": return (rowData, index, data) -> - data[index] = new Date(rowData.getTimestamp(index, 6).getMillisecond()); + data[index] = + org.apache.hadoop.hive.common.type.Date.ofEpochMilli( + rowData.getTimestamp(index, 6).getMillisecond()); case "BINARY": return (rowData, index, data) -> data[index] = new BytesWritable(rowData.getBinary(index)); diff --git a/chunjun-connectors/chunjun-connector-hdfs/src/main/java/com/dtstack/chunjun/connector/hdfs/util/HdfsUtil.java b/chunjun-connectors/chunjun-connector-hdfs/src/main/java/com/dtstack/chunjun/connector/hdfs/util/HdfsUtil.java index 6757e0a625..a93a2a059a 100644 --- a/chunjun-connectors/chunjun-connector-hdfs/src/main/java/com/dtstack/chunjun/connector/hdfs/util/HdfsUtil.java +++ b/chunjun-connectors/chunjun-connector-hdfs/src/main/java/com/dtstack/chunjun/connector/hdfs/util/HdfsUtil.java @@ -159,13 +159,13 @@ public static ObjectInspector columnTypeToObjectInspetor(ColumnType columnType) case TIMESTAMP: objectInspector = ObjectInspectorFactory.getReflectionObjectInspector( - java.sql.Timestamp.class, + org.apache.hadoop.hive.common.type.Timestamp.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); break; case DATE: objectInspector = ObjectInspectorFactory.getReflectionObjectInspector( - java.sql.Date.class, + org.apache.hadoop.hive.common.type.Date.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); break; case STRING: From 99f35a28d241023368d0555e598de699ba7d9e87 Mon Sep 17 00:00:00 2001 From: OT-QX <43375919+ll076110@users.noreply.github.com> Date: Tue, 20 Dec 2022 10:23:03 +0800 Subject: [PATCH 19/41] [hotfix-#1449][ClickHouse] fix ClickHouse source null pointer problem (#1450) Co-authored-by: OT-XY (cherry picked from commit bc190cef1cc6588d0747b3be95a1d0063879876e) --- .../clickhouse/source/ClickhouseInputFormat.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/chunjun-connectors/chunjun-connector-clickhouse/src/main/java/com/dtstack/chunjun/connector/clickhouse/source/ClickhouseInputFormat.java b/chunjun-connectors/chunjun-connector-clickhouse/src/main/java/com/dtstack/chunjun/connector/clickhouse/source/ClickhouseInputFormat.java index f41b6d1a40..cc8da2cff2 100644 --- a/chunjun-connectors/chunjun-connector-clickhouse/src/main/java/com/dtstack/chunjun/connector/clickhouse/source/ClickhouseInputFormat.java +++ b/chunjun-connectors/chunjun-connector-clickhouse/src/main/java/com/dtstack/chunjun/connector/clickhouse/source/ClickhouseInputFormat.java @@ -36,13 +36,13 @@ public class ClickhouseInputFormat extends JdbcInputFormat { @Override public void openInternal(InputSplit inputSplit) { - JdbcInputSplit jdbcInputSplit = (JdbcInputSplit) inputSplit; - initMetric(jdbcInputSplit); - if (!canReadData(jdbcInputSplit)) { + currentJdbcInputSplit = (JdbcInputSplit) inputSplit; + initMetric(currentJdbcInputSplit); + if (!canReadData(currentJdbcInputSplit)) { LOG.warn( "Not read data when the start location are equal to end location, start = {}, end = {}", - jdbcInputSplit.getStartLocation(), - jdbcInputSplit.getEndLocation()); + currentJdbcInputSplit.getStartLocation(), + currentJdbcInputSplit.getEndLocation()); hasNext = false; return; } @@ -50,9 +50,9 @@ public void openInternal(InputSplit inputSplit) { String querySQL = null; try { dbConn = getConnection(); - querySQL = buildQuerySql(jdbcInputSplit); + querySQL = buildQuerySql(currentJdbcInputSplit); jdbcConf.setQuerySql(querySQL); - executeQuery(jdbcInputSplit.getStartLocation()); + executeQuery(currentJdbcInputSplit.getStartLocation()); // 增量任务 needUpdateEndLocation = jdbcConf.isIncrement() && !jdbcConf.isPolling() && !jdbcConf.isUseMaxFunc(); From 1c893db37ae5d0b07c00a3edabb91a0e73a157d5 Mon Sep 17 00:00:00 2001 From: superPiKaChu <1623857502@qq.com> Date: Wed, 21 Dec 2022 10:35:40 +0800 Subject: [PATCH 20/41] [Feature-#1451] add column replace for hbase column (#1452) (cherry picked from commit d3db08bc771c762cdb3edbb36a4d84f3a6b914e2) --- .../connector/hbase/FunctionParser.java | 11 +++ .../hbase/converter/HBaseColumnConverter.java | 77 +++++++++++++++++-- 2 files changed, 83 insertions(+), 5 deletions(-) diff --git a/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/FunctionParser.java b/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/FunctionParser.java index de0746c7b0..31b16c2d71 100644 --- a/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/FunctionParser.java +++ b/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/FunctionParser.java @@ -172,4 +172,15 @@ public static String replaceColToStringFunc(String express) { return express; } + + public static List getRegexColumnName(String qualifier) { + Matcher matcher = COL_PATTERN.matcher(qualifier); + ArrayList columnQualifier = new ArrayList<>(); + while (matcher.find()) { + String columnGroup = matcher.group(); + String column = columnGroup.substring(2, columnGroup.length() - 1); + columnQualifier.add(column); + } + return columnQualifier; + } } diff --git a/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/converter/HBaseColumnConverter.java b/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/converter/HBaseColumnConverter.java index 646d9a6bc0..aedcc3303c 100644 --- a/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/converter/HBaseColumnConverter.java +++ b/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/converter/HBaseColumnConverter.java @@ -60,8 +60,10 @@ import java.text.SimpleDateFormat; import java.time.LocalTime; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; @@ -96,6 +98,8 @@ public class HBaseColumnConverter private final List columnNames = new ArrayList<>(); + private final List columnNamesWithoutcf = new ArrayList<>(); + private final String encoding; private final HBaseConf hBaseConf; @@ -106,7 +110,13 @@ public class HBaseColumnConverter private final List fieldList; - private final byte[][][] familyAndQualifier; + private byte[][][] familyAndQualifier; + + private final byte[][][] familyAndQualifierBack; + + private final ArrayList> columnConfig; + + private final HashSet columnConfigIndex; public HBaseColumnConverter(HBaseConf hBaseConf, RowType rowType) { super(rowType, hBaseConf); @@ -123,6 +133,9 @@ public HBaseColumnConverter(HBaseConf hBaseConf, RowType rowType) { createExternalConverter(rowType.getTypeAt(i)), rowType.getTypeAt(i))); } this.familyAndQualifier = new byte[rowType.getFieldCount()][][]; + this.familyAndQualifierBack = new byte[rowType.getFieldCount()][][]; + this.columnConfig = new ArrayList<>(rowType.getFieldCount()); + this.columnConfigIndex = new HashSet<>(rowType.getFieldCount()); for (int i = 0; i < hBaseConf.getColumn().size(); i++) { FieldConf fieldConf = hBaseConf.getColumn().get(i); String name = fieldConf.getName(); @@ -131,20 +144,26 @@ public HBaseColumnConverter(HBaseConf hBaseConf, RowType rowType) { if (cfAndQualifier.length == 2 && StringUtils.isNotBlank(cfAndQualifier[0]) && StringUtils.isNotBlank(cfAndQualifier[1])) { - + columnNamesWithoutcf.add(cfAndQualifier[1]); byte[][] qualifierKeys = new byte[2][]; - qualifierKeys[0] = Bytes.toBytes(cfAndQualifier[0]); - qualifierKeys[1] = Bytes.toBytes(cfAndQualifier[1]); + qualifierKeys[0] = Bytes.toBytes(cfAndQualifier[0]); // 列族 + qualifierKeys[1] = Bytes.toBytes(cfAndQualifier[1]); // 列名 + columnConfig.add(i, handleColumnConfig(cfAndQualifier[1])); familyAndQualifier[i] = qualifierKeys; + familyAndQualifierBack[i] = Arrays.copyOf(qualifierKeys, qualifierKeys.length); } else if (KEY_ROW_KEY.equals(name)) { rowKeyIndex = i; + columnNamesWithoutcf.add(KEY_ROW_KEY); + columnConfig.add(i, null); } else if (!StringUtils.isBlank(fieldConf.getValue())) { familyAndQualifier[i] = new byte[2][]; + familyAndQualifierBack[i] = new byte[2][]; } else { throw new IllegalArgumentException( "hbase 中，column 的列配置格式应该是：列族:列名. 您配置的列错误：" + name); } } + fieldList = hBaseConf.getColumnMetaInfos(); this.hBaseConf = hBaseConf; @@ -193,10 +212,28 @@ public Mutation toExternal(RowData rowData, Mutation output) throws Exception { } for (int i = 0; i < rowData.getArity(); i++) { - if (rowKeyIndex == i) { + if (rowKeyIndex == i || columnConfigIndex.contains(i)) { continue; } + if (columnConfig.get(i) != null) { + byte[][] qualifier = familyAndQualifier[i]; + qualifier[1] = + fillColumnConfig(new String(qualifier[1]), columnConfig.get(i), rowData); + familyAndQualifier[i] = qualifier; + } toExternalConverters.get(i).serialize(rowData, i, put); + if (i == rowData.getArity() - 1) { + for (int x = 0; x < familyAndQualifierBack.length; x++) { + familyAndQualifier[x] = + Arrays.copyOf( + familyAndQualifierBack[x], familyAndQualifierBack[x].length); + if (x + 1 < familyAndQualifierBack.length + && familyAndQualifierBack[x + 1] == null) { + familyAndQualifier[x + 1] = null; + x = x + 1; + } + } + } } return put; } @@ -594,4 +631,34 @@ private static SimpleDateFormat getSimpleDateFormat(String sign) { } return format; } + + private HashMap handleColumnConfig(String qualifier) { + HashMap columnConfigMap = new HashMap<>(columnNames.size()); + List regexColumnNameList = FunctionParser.getRegexColumnName(qualifier); + if (!regexColumnNameList.isEmpty()) { + for (int i = 0; i < regexColumnNameList.size(); i++) { + columnConfigMap.put( + regexColumnNameList.get(i), + columnNamesWithoutcf.indexOf(regexColumnNameList.get(i))); + columnConfigIndex.add(columnNamesWithoutcf.indexOf(regexColumnNameList.get(i))); + } + } else { + columnConfigMap = null; + } + return columnConfigMap; + } + + private byte[] fillColumnConfig( + String columnValue, HashMap columnConfigMap, RowData rowData) { + List regexColumnNameList = FunctionParser.getRegexColumnName(columnValue); + for (String regrexColumn : regexColumnNameList) { + Integer columnIndex = columnConfigMap.get(regrexColumn); + columnValue = + StringUtils.replace( + columnValue, + "$(" + regrexColumn + ")", + rowData.getString(columnIndex).toString()); + } + return Bytes.toBytes(columnValue); + } } From 4720a0adfe627c34ce680fcdc46c502bbb2ca60d Mon Sep 17 00:00:00 2001 From: superPiKaChu <1623857502@qq.com> Date: Tue, 3 Jan 2023 09:06:14 +0800 Subject: [PATCH 21/41] [feat-#1455] fix hbase bug & add hbase ttl (#1456) * [feat-##1455] fix hbase bug & add hbase ttl * [feat-##1455] update ttl value set (cherry picked from commit 3c72eaeaa65be5a3617be61189a3ccf16515a99b) --- .../connector/hbase/conf/HBaseConf.java | 9 +++++++++ .../hbase/converter/HBaseColumnConverter.java | 18 +++++++++++------- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/conf/HBaseConf.java b/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/conf/HBaseConf.java index 318511bec4..08714c8c7f 100644 --- a/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/conf/HBaseConf.java +++ b/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/conf/HBaseConf.java @@ -61,6 +61,7 @@ public class HBaseConf extends ChunJunCommonConf { private String rowkeyExpress; private Integer versionColumnIndex; private String versionColumnValue; + private Long ttl; public String getEncoding() { return encoding; @@ -181,4 +182,12 @@ public List getColumnMetaInfos() { public void setColumnMetaInfos(List columnMetaInfos) { this.columnMetaInfos = columnMetaInfos; } + + public void setTTL(Long ttl) { + this.ttl = ttl; + } + + public Long getTtl() { + return ttl; + } } diff --git a/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/converter/HBaseColumnConverter.java b/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/converter/HBaseColumnConverter.java index aedcc3303c..132fd94cd9 100644 --- a/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/converter/HBaseColumnConverter.java +++ b/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/converter/HBaseColumnConverter.java @@ -66,6 +66,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import static com.dtstack.chunjun.connector.hbase.HBaseTypeUtils.MAX_TIMESTAMP_PRECISION; import static com.dtstack.chunjun.connector.hbase.HBaseTypeUtils.MIN_TIMESTAMP_PRECISION; @@ -211,6 +212,9 @@ public Mutation toExternal(RowData rowData, Mutation output) throws Exception { put = new Put(rowkey, version); } + put.setTTL( + Optional.ofNullable(hBaseConf.getTtl()).orElseGet(() -> (long) Integer.MAX_VALUE)); + for (int i = 0; i < rowData.getArity(); i++) { if (rowKeyIndex == i || columnConfigIndex.contains(i)) { continue; @@ -224,13 +228,13 @@ public Mutation toExternal(RowData rowData, Mutation output) throws Exception { toExternalConverters.get(i).serialize(rowData, i, put); if (i == rowData.getArity() - 1) { for (int x = 0; x < familyAndQualifierBack.length; x++) { - familyAndQualifier[x] = - Arrays.copyOf( - familyAndQualifierBack[x], familyAndQualifierBack[x].length); - if (x + 1 < familyAndQualifierBack.length - && familyAndQualifierBack[x + 1] == null) { - familyAndQualifier[x + 1] = null; - x = x + 1; + if (familyAndQualifierBack[x] == null) { + familyAndQualifier[x] = null; + } else { + familyAndQualifier[x] = + Arrays.copyOf( + familyAndQualifierBack[x], + familyAndQualifierBack[x].length); } } } From f2d1fc8bc5a2fe1c71f18b63913bee0098b1b815 Mon Sep 17 00:00:00 2001 From: FlechazoW <35768015+FlechazoW@users.noreply.github.com> Date: Thu, 5 Jan 2023 16:23:48 +0800 Subject: [PATCH 22/41] [BugFix][Postgres][#1463] The cast is bad, only when 'rowConverter' instance of the 'JdbcColumnConverter' should cast. (#1464) (cherry picked from commit 1d87b291d9029d8bf9b971570034eb08e534ae46) --- .../connector/postgresql/sink/PostgresOutputFormat.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/sink/PostgresOutputFormat.java b/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/sink/PostgresOutputFormat.java index a5dd6f4601..8e3f69629a 100644 --- a/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/sink/PostgresOutputFormat.java +++ b/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/sink/PostgresOutputFormat.java @@ -88,10 +88,12 @@ protected void openInternal(int taskNumber, int numTasks) { LOG.info("write sql:{}", copySql); } checkUpsert(); - if (jdbcDialect.dialectName().equals("PostgreSQL")) { - ((PostgresqlColumnConverter) rowConverter).setConnection((BaseConnection) dbConn); + if (rowConverter instanceof JdbcColumnConverter) { + if (jdbcDialect.dialectName().equals("PostgresSQL")) { + ((PostgresqlColumnConverter) rowConverter).setConnection((BaseConnection) dbConn); + } + ((PostgresqlColumnConverter) rowConverter).setFieldTypeList(columnTypeList); } - ((PostgresqlColumnConverter) rowConverter).setFieldTypeList(columnTypeList); } catch (SQLException sqe) { throw new IllegalArgumentException("checkUpsert() failed.", sqe); } From 17f9a341b76a563fba6870215513fd2931cc17bb Mon Sep 17 00:00:00 2001 From: FlechazoW <35768015+FlechazoW@users.noreply.github.com> Date: Thu, 5 Jan 2023 17:11:13 +0800 Subject: [PATCH 23/41] [BugFix][Jdbc][#1459] Remove the 'checkConnValid', because some connector doesn't support 'isValid'. (#1465) (cherry picked from commit 0db36b8a9cea6136a75b2987ed865480a4db2963) --- .../connector/jdbc/sink/JdbcOutputFormat.java | 25 ------------------- .../postgresql/sink/PostgresOutputFormat.java | 3 ++- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormat.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormat.java index af3e0c0a72..e2db65adf5 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormat.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormat.java @@ -29,7 +29,6 @@ import com.dtstack.chunjun.enums.EWriteMode; import com.dtstack.chunjun.enums.Semantic; import com.dtstack.chunjun.sink.format.BaseRichOutputFormat; -import com.dtstack.chunjun.throwable.ChunJunRuntimeException; import com.dtstack.chunjun.throwable.WriteRecordException; import com.dtstack.chunjun.util.ExceptionUtil; import com.dtstack.chunjun.util.GsonUtil; @@ -37,7 +36,6 @@ import org.apache.flink.table.data.GenericRowData; import org.apache.flink.table.data.RowData; -import org.apache.flink.util.FlinkRuntimeException; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -181,29 +179,6 @@ protected void writeMultipleRecordsInternal() throws Exception { } } - @Override - public synchronized void writeRecord(RowData rowData) { - checkConnValid(); - super.writeRecord(rowData); - } - - public void checkConnValid() { - try { - LOG.debug("check db connection valid.."); - if (!dbConn.isValid(10)) { - if (Semantic.EXACTLY_ONCE == semantic) { - throw new FlinkRuntimeException( - "jdbc connection is valid!work's semantic is ExactlyOnce.To prevent data loss,we don't try to reopen the connection"); - } - LOG.info("db connection reconnect.."); - dbConn = getConnection(); - stmtProxy.reOpen(dbConn); - } - } catch (Exception e) { - throw new ChunJunRuntimeException("failed to check jdbcConnection valid", e); - } - } - @Override public void preCommit() throws Exception { if (jdbcConf.getRestoreColumnIndex() > -1) { diff --git a/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/sink/PostgresOutputFormat.java b/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/sink/PostgresOutputFormat.java index 8e3f69629a..f39d02b38e 100644 --- a/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/sink/PostgresOutputFormat.java +++ b/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/sink/PostgresOutputFormat.java @@ -90,7 +90,8 @@ protected void openInternal(int taskNumber, int numTasks) { checkUpsert(); if (rowConverter instanceof JdbcColumnConverter) { if (jdbcDialect.dialectName().equals("PostgresSQL")) { - ((PostgresqlColumnConverter) rowConverter).setConnection((BaseConnection) dbConn); + ((PostgresqlColumnConverter) rowConverter) + .setConnection((BaseConnection) dbConn); } ((PostgresqlColumnConverter) rowConverter).setFieldTypeList(columnTypeList); } From 63a396e7da374c967102591a1bba6124204f8c4f Mon Sep 17 00:00:00 2001 From: Zhang Changjun <479224070@qq.com> Date: Tue, 10 Jan 2023 10:38:03 +0800 Subject: [PATCH 24/41] [Bug- #1473 ][chunjun-clients] fix Logic error in confProp parameter processing.(#1477) [Bug- #1473 ][chunjun-clients] fix Logic error in confProp parameter processing (cherry picked from commit df98c321e64142ae373c73f7d011200a13ec93aa) --- .../yarn/YarnPerJobClusterClientHelper.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/chunjun-clients/src/main/java/com/dtstack/chunjun/client/yarn/YarnPerJobClusterClientHelper.java b/chunjun-clients/src/main/java/com/dtstack/chunjun/client/yarn/YarnPerJobClusterClientHelper.java index f2e96e0c3d..f44a2524d9 100644 --- a/chunjun-clients/src/main/java/com/dtstack/chunjun/client/yarn/YarnPerJobClusterClientHelper.java +++ b/chunjun-clients/src/main/java/com/dtstack/chunjun/client/yarn/YarnPerJobClusterClientHelper.java @@ -29,6 +29,7 @@ import org.apache.flink.client.program.ClusterClientProvider; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.JobManagerOptions; +import org.apache.flink.configuration.MemorySize; import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; @@ -182,20 +183,21 @@ private ClusterSpecification createClusterSpecification(JobDeployer jobDeployer) jobManagerMemoryMb = Math.max( MIN_JM_MEMORY, - ValueUtil.getInt( - conProp.getProperty( - JobManagerOptions.TOTAL_PROCESS_MEMORY.key()))); - jobManagerMemoryMb = jobManagerMemoryMb >> 20; + MemorySize.parse( + conProp.getProperty( + JobManagerOptions.TOTAL_PROCESS_MEMORY + .key())) + .getMebiBytes()); } if (conProp.containsKey(TaskManagerOptions.TOTAL_PROCESS_MEMORY.key())) { taskManagerMemoryMb = Math.max( MIN_TM_MEMORY, - ValueUtil.getInt( - conProp.getProperty( - TaskManagerOptions.TOTAL_PROCESS_MEMORY.key()))); - - taskManagerMemoryMb = taskManagerMemoryMb >> 20; + MemorySize.parse( + conProp.getProperty( + TaskManagerOptions.TOTAL_PROCESS_MEMORY + .key())) + .getMebiBytes()); } if (conProp.containsKey(NUM_TASK_SLOTS.key())) { slotsPerTaskManager = ValueUtil.getInt(conProp.get(NUM_TASK_SLOTS.key())); From b9e2a342466546ea083d5a596840782e043d6e24 Mon Sep 17 00:00:00 2001 From: wangJm Date: Wed, 11 Jan 2023 10:11:39 +0800 Subject: [PATCH 25/41] [feat-#1470][ftp] improvement ftp read/write performance (#1483) (cherry picked from commit 9d06b7d42f99abb7263d697277782a82b0fb3368) --- .../chunjun/connector/ftp/client/Data.java | 13 +- .../connector/ftp/client/FileUtil.java | 132 ----------- .../connector/ftp/client/ZipInputStream.java | 5 +- .../ftp/client/excel/ExcelReaderExecutor.java | 2 +- .../excel/ExcelSubExceptionCarrier.java | 2 +- .../chunjun/connector/ftp/conf/FtpConfig.java | 44 +++- .../ftp/converter/FtpColumnConverter.java | 44 ++-- .../ftp/converter/FtpRawTypeConverter.java | 5 - .../ftp/converter/FtpRowConverter.java | 5 - .../chunjun/connector/ftp/enums/FileType.java | 3 +- .../extend/converter/IColumnConverter.java} | 22 +- .../ftp/{client => extend/ftp}/File.java | 2 +- .../ftp/extend/ftp/FtpParseException.java} | 32 +-- .../{format => extend/ftp}/IFormatConfig.java | 41 +++- .../{handler => extend/ftp}/IFtpHandler.java | 39 ++-- .../ftp/concurrent/ConcurrentFileSplit.java | 46 ++++ .../ftp/concurrent}/FtpFileSplit.java | 10 +- .../ftp}/format/IFileReadFormat.java | 5 +- .../DTFtpHandler.java} | 39 ++-- .../connector/ftp/handler/FtpHandler.java | 130 ++++++++--- .../ftp/handler/FtpHandlerFactory.java | 8 +- .../connector/ftp/handler/Position.java | 10 +- .../connector/ftp/handler/SftpHandler.java | 89 +++++-- .../{format => iformat}/CsvFileFormat.java | 6 +- .../{format => iformat}/ExcelFileFormat.java | 16 +- .../{format => iformat}/IFormatFactory.java | 5 +- .../{format => iformat}/TextFileFormat.java | 8 +- .../connector/ftp/options/FtpOptions.java | 17 +- .../ftp/sink/FtpDynamicTableSink.java | 5 - .../connector/ftp/sink/FtpOutputFormat.java | 82 ++++--- .../ftp/sink/FtpOutputFormatBuilder.java | 13 +- .../connector/ftp/sink/FtpSinkFactory.java | 5 - .../ftp/source/FtpDynamicTableSource.java | 5 - .../connector/ftp/source/FtpFileReader.java | 219 ++++++++++++++---- .../connector/ftp/source/FtpInputFormat.java | 91 ++++---- .../ftp/source/FtpInputFormatBuilder.java | 1 - .../connector/ftp/source/FtpInputSplit.java | 2 + .../ftp/source/FtpInputStreamProxy.java | 107 +++++++++ .../ftp/source/FtpSourceFactory.java | 23 +- .../ftp/spliter/ConcurrentCsvSplit.java | 122 ++++++++++ .../spliter/ConcurrentFileSplitFactory.java | 67 ++++++ .../spliter/ConcurrentZipCompressSplit.java | 64 +++++ .../ftp/spliter/DefaultFileSplit.java | 68 ++++++ .../ftp/table/FtpDynamicTableFactory.java | 11 +- .../ftp/format/TextFileFormatTest.java | 68 ------ ...ChunjunFlinkStandaloneTestEnvironment.java | 2 +- 46 files changed, 1162 insertions(+), 573 deletions(-) rename chunjun-connectors/chunjun-connector-ftp/src/{test/java/com/dtstack/chunjun/connector/ftp/format/IFormatConfigTest.java => main/java/com/dtstack/chunjun/connector/ftp/extend/converter/IColumnConverter.java} (62%) rename chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/{client => extend/ftp}/File.java (98%) rename chunjun-connectors/chunjun-connector-ftp/src/{test/java/com/dtstack/chunjun/connector/ftp/client/excel/ExcelSubExceptionCarrierTest.java => main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/FtpParseException.java} (56%) rename chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/{format => extend/ftp}/IFormatConfig.java (73%) rename chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/{handler => extend/ftp}/IFtpHandler.java (79%) create mode 100644 chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/concurrent/ConcurrentFileSplit.java rename chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/{source => extend/ftp/concurrent}/FtpFileSplit.java (93%) rename chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/{ => extend/ftp}/format/IFileReadFormat.java (92%) rename chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/{client/excel/ExcelReaderExceptionHandler.java => handler/DTFtpHandler.java} (54%) rename chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/{format => iformat}/CsvFileFormat.java (91%) rename chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/{format => iformat}/ExcelFileFormat.java (91%) rename chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/{format => iformat}/IFormatFactory.java (91%) rename chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/{format => iformat}/TextFileFormat.java (86%) create mode 100644 chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpInputStreamProxy.java create mode 100644 chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/spliter/ConcurrentCsvSplit.java create mode 100644 chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/spliter/ConcurrentFileSplitFactory.java create mode 100644 chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/spliter/ConcurrentZipCompressSplit.java create mode 100644 chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/spliter/DefaultFileSplit.java delete mode 100644 chunjun-connectors/chunjun-connector-ftp/src/test/java/com/dtstack/chunjun/connector/ftp/format/TextFileFormatTest.java diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/Data.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/Data.java index 2b79fca4b5..facdd300a2 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/Data.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/Data.java @@ -18,22 +18,33 @@ package com.dtstack.chunjun.connector.ftp.client; +import com.dtstack.chunjun.connector.ftp.extend.ftp.FtpParseException; import com.dtstack.chunjun.connector.ftp.handler.Position; -/** return from ftpSeqBufferedReader contains line and position */ public class Data { private String[] data; private Position position; + private FtpParseException exception; public Data(String[] data, Position position) { this.data = data; this.position = position; } + public Data(String[] data, Position position, FtpParseException exception) { + this.data = data; + this.position = position; + this.exception = exception; + } + public String[] getData() { return data; } + public FtpParseException getException() { + return exception; + } + public Position getPosition() { return position; } diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/FileUtil.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/FileUtil.java index 194f92b43c..e3fdfb3b17 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/FileUtil.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/FileUtil.java @@ -18,146 +18,14 @@ package com.dtstack.chunjun.connector.ftp.client; -import com.dtstack.chunjun.connector.ftp.conf.FtpConfig; -import com.dtstack.chunjun.connector.ftp.handler.FtpHandler; -import com.dtstack.chunjun.connector.ftp.handler.IFtpHandler; -import com.dtstack.chunjun.connector.ftp.source.FtpFileSplit; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.Charset; -import java.util.List; -import java.util.Locale; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; - public class FileUtil { private static final Logger LOG = LoggerFactory.getLogger(FileUtil.class); - public static void addCompressFile( - IFtpHandler ftpHandler, - String filePath, - FtpConfig ftpConfig, - List fileList) - throws IOException { - if ("ZIP".equals(ftpConfig.getCompressType().toUpperCase(Locale.ENGLISH))) { - try (ZipInputStream zipInputStream = - new ZipInputStream( - ftpHandler.getInputStream(filePath), - Charset.forName(ftpConfig.encoding))) { - ZipEntry zipEntry; - while ((zipEntry = zipInputStream.getNextEntry()) != null) { - fileList.add( - new FtpFileSplit( - 0, - zipEntry.getSize(), - filePath, - zipEntry.getName(), - ftpConfig.getCompressType())); - } - closeWithFtpHandler(ftpHandler, LOG); - } - } else { - throw new RuntimeException("not support compressType " + ftpConfig.getCompressType()); - } - } - - public static void closeWithFtpHandler(IFtpHandler ftpHandler, Logger log) { - if (ftpHandler instanceof FtpHandler) { - try { - ((FtpHandler) ftpHandler).getFtpClient().completePendingCommand(); - } catch (Exception e) { - log.warn("FTPClient completePendingCommand has error ->", e); - try { - ftpHandler.logoutFtpServer(); - } catch (Exception exception) { - log.warn("FTPClient logout has error ->", exception); - } - } - } - } - public static String getFilename(String filepath) { String[] paths = filepath.split("/"); return paths[paths.length - 1]; } - - /** analyse file */ - public static void addFile( - IFtpHandler ftpHandler, - String filePath, - FtpConfig ftpConfig, - List fileList) - throws Exception { - long maxFetchSize = ftpConfig.getMaxFetchSize(); - - // fetchSize should bigger than 1M - maxFetchSize = Math.max(maxFetchSize, 1024 * 1024); - - long currentFileSize = ftpHandler.getFileSize(filePath); - int parallelism = ftpConfig.getParallelism(); - - String filename = getFilename(filePath); - - // do not split excel - if (ftpConfig.getFileType() == null - || ftpConfig.getFileType().equals("excel") - || ftpConfig.getFileType().equals("custom")) { - FtpFileSplit ftpFileSplit = new FtpFileSplit(0, currentFileSize, filePath, filename); - fileList.add(ftpFileSplit); - return; - } - - // split file - if (maxFetchSize < currentFileSize) { - int perSplit = Math.min((int) currentFileSize / parallelism, Integer.MAX_VALUE); - long startPosition = 0; - long endPosition = startPosition + perSplit; - - while (endPosition <= currentFileSize) { - if (endPosition == currentFileSize) { - FtpFileSplit ftpFileSplit = - new FtpFileSplit(startPosition, endPosition, filePath, filename); - fileList.add(ftpFileSplit); - break; - } - - InputStream input = ftpHandler.getInputStreamByPosition(filePath, endPosition); - char c = ' '; - - while (c != '\n') { - c = (char) input.read(); - endPosition += 1; - } - FtpFileSplit ftpFileSplit = - new FtpFileSplit(startPosition, endPosition, filePath, filename); - fileList.add(ftpFileSplit); - - LOG.info( - String.format( - "build file split, filename: %s, startPosition: %d, endPosition: %d", - filePath, startPosition, endPosition)); - - startPosition = endPosition; - endPosition = startPosition + perSplit; - } - - if (startPosition != currentFileSize) { - FtpFileSplit ftpFileSplit = - new FtpFileSplit(startPosition, currentFileSize, filePath, filename); - fileList.add(ftpFileSplit); - - LOG.info( - String.format( - "build file split, filename: %s, startPosition: %d, endPosition: %d", - filePath, startPosition, currentFileSize)); - } - } else { - FtpFileSplit ftpFileSplit = new FtpFileSplit(0, currentFileSize, filePath, filename); - fileList.add(ftpFileSplit); - } - } } diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/ZipInputStream.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/ZipInputStream.java index 44e6469598..940b84086b 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/ZipInputStream.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/ZipInputStream.java @@ -29,13 +29,12 @@ import java.util.List; import java.util.zip.ZipEntry; -/** zip文件流 如果fileNameList不为空 只会读取fileNameList里的文件* */ public class ZipInputStream extends InputStream { private static final Logger LOG = LoggerFactory.getLogger(ZipInputStream.class); - private java.util.zip.ZipInputStream zipInputStream; + private final java.util.zip.ZipInputStream zipInputStream; + private final List fileNameList; private ZipEntry currentZipEntry; - private List fileNameList; public ZipInputStream(InputStream in) { this.zipInputStream = new java.util.zip.ZipInputStream(in); diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/excel/ExcelReaderExecutor.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/excel/ExcelReaderExecutor.java index f985cd3dc4..eb019fde3f 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/excel/ExcelReaderExecutor.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/excel/ExcelReaderExecutor.java @@ -23,7 +23,7 @@ public class ExcelReaderExecutor implements Runnable { private final ExcelReader reader; - private final ExcelSubExceptionCarrier ec; + private ExcelSubExceptionCarrier ec; public ExcelReaderExecutor(ExcelReader reader, ExcelSubExceptionCarrier ec) { this.reader = reader; diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/excel/ExcelSubExceptionCarrier.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/excel/ExcelSubExceptionCarrier.java index b7098af6a7..6e6c4fe48a 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/excel/ExcelSubExceptionCarrier.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/excel/ExcelSubExceptionCarrier.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/conf/FtpConfig.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/conf/FtpConfig.java index 1cd3a01f28..da856160d4 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/conf/FtpConfig.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/conf/FtpConfig.java @@ -19,6 +19,7 @@ package com.dtstack.chunjun.connector.ftp.conf; import com.dtstack.chunjun.conf.BaseFileConf; +import com.dtstack.chunjun.connector.ftp.enums.FileType; import com.dtstack.chunjun.constants.ConstantValue; import java.util.Map; @@ -47,6 +48,9 @@ public class FtpConfig extends BaseFileConf { private String ftpFileName; + /** 批量写入数据太大，会导致ftp协议缓冲区报错, 批量写入默认值设置小点 */ + private long nextCheckRows = 100; + public String encoding = "UTF-8"; /** 空值替换 */ @@ -58,12 +62,34 @@ public class FtpConfig extends BaseFileConf { /** User defined format class name */ private String customFormatClassName; + /** User defined split class name */ + private String customConcurrentFileSplitClassName; + + /* 行分隔符 */ + private String columnDelimiter = "\n"; + /** Get the specified fileReadClient according to the filetype * */ - public String fileType; + public String fileType = FileType.TXT.name(); /** 压缩格式 * */ public String compressType; + public String getColumnDelimiter() { + return columnDelimiter; + } + + public void setColumnDelimiter(String columnDelimiter) { + this.columnDelimiter = columnDelimiter; + } + + public String getCustomConcurrentFileSplitClassName() { + return customConcurrentFileSplitClassName; + } + + public void setCustomConcurrentFileSplitClassName(String customConcurrentFileSplitClassName) { + this.customConcurrentFileSplitClassName = customConcurrentFileSplitClassName; + } + public String getCustomFormatClassName() { return customFormatClassName; } @@ -238,6 +264,16 @@ public void setMaxFetchSize(long fetchSize) { this.maxFetchSize = fetchSize; } + @Override + public long getNextCheckRows() { + return nextCheckRows; + } + + @Override + public void setNextCheckRows(long nextCheckRows) { + this.nextCheckRows = nextCheckRows; + } + public long getMaxFetchSize() { return this.maxFetchSize; } @@ -259,10 +295,16 @@ public String toString() { .add("listHiddenFiles=" + listHiddenFiles) .add("maxFetchSize=" + maxFetchSize) .add("ftpFileName='" + ftpFileName + "'") + .add("nextCheckRows=" + nextCheckRows) .add("encoding='" + encoding + "'") .add("nullIsReplacedWithValue=" + nullIsReplacedWithValue) .add("fileConfig=" + fileConfig) .add("customFormatClassName='" + customFormatClassName + "'") + .add( + "customConcurrentFileSplitClassName='" + + customConcurrentFileSplitClassName + + "'") + .add("columnDelimiter='" + columnDelimiter + "'") .add("fileType='" + fileType + "'") .add("compressType='" + compressType + "'") .toString(); diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/converter/FtpColumnConverter.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/converter/FtpColumnConverter.java index ed1af80781..0ec710caa3 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/converter/FtpColumnConverter.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/converter/FtpColumnConverter.java @@ -18,7 +18,6 @@ package com.dtstack.chunjun.connector.ftp.converter; -import com.dtstack.chunjun.conf.FieldConf; import com.dtstack.chunjun.connector.ftp.conf.FtpConfig; import com.dtstack.chunjun.converter.AbstractRowConverter; import com.dtstack.chunjun.converter.IDeserializationConverter; @@ -47,26 +46,21 @@ import java.util.ArrayList; import java.util.List; -/** - * @program chunjun - * @author: xiuzhu - * @create: 2021/06/19 - */ -public class FtpColumnConverter extends AbstractRowConverter { +public class FtpColumnConverter + extends AbstractRowConverter { private final FtpConfig ftpConfig; public FtpColumnConverter(RowType rowType, FtpConfig ftpConfig) { - super(rowType); + super(rowType, ftpConfig); this.ftpConfig = ftpConfig; for (int i = 0; i < rowType.getFieldCount(); i++) { - FieldConf fieldConf = ftpConfig.getColumn().get(i); toInternalConverters.add( wrapIntoNullableInternalConverter( createInternalConverter(rowType.getTypeAt(i)))); toExternalConverters.add( wrapIntoNullableExternalConverter( - createExternalConverter(fieldConf), fieldConf)); + createExternalConverter(rowType.getTypeAt(i)), rowType.getTypeAt(i))); } } @@ -96,7 +90,7 @@ public String toExternal(RowData rowData, String output) throws Exception { StringBuilder sb = new StringBuilder(128); List columnData = new ArrayList<>(ftpConfig.getColumn().size()); - for (int index = 0; index < rowData.getArity(); index++) { + for (int index = 0; index < toExternalConverters.size(); index++) { toExternalConverters.get(index).serialize(rowData, index, columnData); if (index != 0) { sb.append(ftpConfig.getFieldDelimiter()); @@ -109,7 +103,7 @@ public String toExternal(RowData rowData, String output) throws Exception { @Override @SuppressWarnings("unchecked") protected ISerializationConverter> wrapIntoNullableExternalConverter( - ISerializationConverter serializationConverter, FieldConf fieldConf) { + ISerializationConverter serializationConverter, LogicalType logicalType) { return (rowData, index, list) -> { if (rowData == null || rowData.isNullAt(index)) { list.add(index, null); @@ -119,6 +113,7 @@ protected ISerializationConverter> wrapIntoNullableExternalConverte }; } + @Override protected IDeserializationConverter createInternalConverter(LogicalType type) { switch (type.getTypeRoot()) { case BOOLEAN: @@ -159,8 +154,27 @@ protected IDeserializationConverter createInternalConverter(LogicalType type) { } @Override - protected ISerializationConverter> createExternalConverter(FieldConf fieldConf) { - return (rowData, index, list) -> - list.add(index, ((ColumnRowData) rowData).getField(index).asString()); + protected ISerializationConverter> createExternalConverter( + LogicalType logicalType) { + switch (logicalType.getTypeRoot()) { + case DATE: + return (rowData, index, list) -> { + if (rowData instanceof ColumnRowData) { + list.add( + index, + ((ColumnRowData) rowData).getField(index).asSqlDate().toString()); + } else { + list.add(index, ((GenericRowData) rowData).getField(index).toString()); + } + }; + default: + return (rowData, index, list) -> { + if (rowData instanceof ColumnRowData) { + list.add(index, ((ColumnRowData) rowData).getField(index).asString()); + } else { + list.add(index, ((GenericRowData) rowData).getField(index).toString()); + } + }; + } } } diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/converter/FtpRawTypeConverter.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/converter/FtpRawTypeConverter.java index 595379ed5f..cac99580fc 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/converter/FtpRawTypeConverter.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/converter/FtpRawTypeConverter.java @@ -27,11 +27,6 @@ import java.util.Locale; -/** - * @program chunjun - * @author: xiuzhu - * @create: 2021/06/28 - */ public class FtpRawTypeConverter { public static DataType apply(String type) throws UnsupportedTypeException { diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/converter/FtpRowConverter.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/converter/FtpRowConverter.java index 1a2292e46c..8dc8d61e21 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/converter/FtpRowConverter.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/converter/FtpRowConverter.java @@ -25,11 +25,6 @@ import org.apache.flink.table.data.RowData; import org.apache.flink.table.types.logical.LogicalType; -/** - * @program chunjun - * @author: xiuzhu - * @create: 2021/06/19 - */ public class FtpRowConverter extends AbstractRowConverter { private DeserializationSchema valueDeserialization; diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/enums/FileType.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/enums/FileType.java index 93786fb8b4..ca0e1698a9 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/enums/FileType.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/enums/FileType.java @@ -27,8 +27,7 @@ public enum FileType { CSV, EXCEL, /** For user-defined file types, the IFileReadFormat interface needs to be implemented.* */ - CUSTOM, - ; + CUSTOM; public static FileType getType(String type) { try { diff --git a/chunjun-connectors/chunjun-connector-ftp/src/test/java/com/dtstack/chunjun/connector/ftp/format/IFormatConfigTest.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/converter/IColumnConverter.java similarity index 62% rename from chunjun-connectors/chunjun-connector-ftp/src/test/java/com/dtstack/chunjun/connector/ftp/format/IFormatConfigTest.java rename to chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/converter/IColumnConverter.java index 4dd2dc2d1f..782d8f36b2 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/test/java/com/dtstack/chunjun/connector/ftp/format/IFormatConfigTest.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/converter/IColumnConverter.java @@ -16,24 +16,10 @@ * limitations under the License. */ -package com.dtstack.chunjun.connector.ftp.format; +package com.dtstack.chunjun.connector.ftp.extend.converter; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import java.io.Serializable; -public class IFormatConfigTest { - - private IFormatConfig iFormatConfigUnderTest; - - @Before - public void setUp() { - iFormatConfigUnderTest = new IFormatConfig(); - } - - @Test - public void testToString() { - Assert.assertTrue( - iFormatConfigUnderTest.toString().contains(IFormatConfig.class.getSimpleName())); - } +public interface IColumnConverter extends Serializable { + T convert(Object source) throws Exception; } diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/File.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/File.java similarity index 98% rename from chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/File.java rename to chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/File.java index 591f30438a..1f5183d0eb 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/File.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/File.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.dtstack.chunjun.connector.ftp.client; +package com.dtstack.chunjun.connector.ftp.extend.ftp; import java.io.Serializable; import java.util.StringJoiner; diff --git a/chunjun-connectors/chunjun-connector-ftp/src/test/java/com/dtstack/chunjun/connector/ftp/client/excel/ExcelSubExceptionCarrierTest.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/FtpParseException.java similarity index 56% rename from chunjun-connectors/chunjun-connector-ftp/src/test/java/com/dtstack/chunjun/connector/ftp/client/excel/ExcelSubExceptionCarrierTest.java rename to chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/FtpParseException.java index 786318f37c..8e52400dea 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/test/java/com/dtstack/chunjun/connector/ftp/client/excel/ExcelSubExceptionCarrierTest.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/FtpParseException.java @@ -16,25 +16,29 @@ * limitations under the License. */ -package com.dtstack.chunjun.connector.ftp.client.excel; +package com.dtstack.chunjun.connector.ftp.extend.ftp; -import org.junit.Before; -import org.junit.Test; +import java.io.IOException; -public class ExcelSubExceptionCarrierTest { +public class FtpParseException extends IOException { + private String content; - private ExcelSubExceptionCarrier excelSubExceptionCarrierUnderTest; - - @Before - public void setUp() { - excelSubExceptionCarrierUnderTest = new ExcelSubExceptionCarrier(); + /** + * @param content 解析的内容 + * @param errMsg 报错原因 + * @param cause + */ + public FtpParseException(String content, String errMsg, Throwable cause) { + super(errMsg, cause); + this.content = content; } - @Test(expected = RuntimeException.class) - public void setExcelSubExceptionCarrierUnderTest() throws Exception { - RuntimeException test = new RuntimeException("Test"); - excelSubExceptionCarrierUnderTest.setThrowable(test); + public String getContent() { + return content; + } - throw excelSubExceptionCarrierUnderTest.getThrowable(); + @Override + public String toString() { + return "FtpParseException{" + "content='" + content + '\'' + '}'; } } diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/format/IFormatConfig.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/IFormatConfig.java similarity index 73% rename from chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/format/IFormatConfig.java rename to chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/IFormatConfig.java index 0ed5fe2173..5fba54bc63 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/format/IFormatConfig.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/IFormatConfig.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.dtstack.chunjun.connector.ftp.format; +package com.dtstack.chunjun.connector.ftp.extend.ftp; import java.io.Serializable; import java.util.Arrays; @@ -27,12 +27,28 @@ public class IFormatConfig implements Serializable { private static final long serialVersionUID = 1L; - private String fieldDelimiter; private String encoding; private String[] fields; private boolean isFirstLineHeader; private Map fileConfig; + private int parallelism; + private long fetchMaxSize; + + /* 列分隔符 */ + private String fieldDelimiter; + + /* 行分隔符 */ + private String columnDelimiter; + + public void setColumnDelimiter(String columnDelimiter) { + this.columnDelimiter = columnDelimiter; + } + + public String getColumnDelimiter() { + return columnDelimiter; + } + public boolean isFirstLineHeader() { return isFirstLineHeader; } @@ -73,14 +89,33 @@ public void setFields(String[] fields) { this.fields = fields; } + public int getParallelism() { + return parallelism; + } + + public void setParallelism(int parallelism) { + this.parallelism = parallelism; + } + + public void setFetchMaxSize(long fetchMaxSize) { + this.fetchMaxSize = fetchMaxSize; + } + + public long getFetchMaxSize() { + return fetchMaxSize; + } + @Override public String toString() { return new StringJoiner(", ", IFormatConfig.class.getSimpleName() + "[", "]") - .add("fieldDelimiter='" + fieldDelimiter + "'") .add("encoding='" + encoding + "'") .add("fields=" + Arrays.toString(fields)) .add("isFirstLineHeader=" + isFirstLineHeader) .add("fileConfig=" + fileConfig) + .add("parallelism=" + parallelism) + .add("fetchMaxSize=" + fetchMaxSize) + .add("fieldDelimiter='" + fieldDelimiter + "'") + .add("columnDelimiter='" + columnDelimiter + "'") .toString(); } } diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/handler/IFtpHandler.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/IFtpHandler.java similarity index 79% rename from chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/handler/IFtpHandler.java rename to chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/IFtpHandler.java index 71956d631d..81ad023f2c 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/handler/IFtpHandler.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/IFtpHandler.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,32 +16,15 @@ * limitations under the License. */ -package com.dtstack.chunjun.connector.ftp.handler; - -import com.dtstack.chunjun.connector.ftp.conf.FtpConfig; +package com.dtstack.chunjun.connector.ftp.extend.ftp; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.List; -/** The skeleton of Ftp Utility class */ public interface IFtpHandler extends AutoCloseable { - /** - * 登录服务器 - * - * @param ftpConfig 连接配置 - */ - void loginFtpServer(FtpConfig ftpConfig); - - /** - * 登出服务器 - * - * @throws IOException logout error - */ - void logoutFtpServer() throws IOException; - /** * 判断给定的目录是否存在 * @@ -56,7 +39,7 @@ public interface IFtpHandler extends AutoCloseable { * @param filePath 要检查的文件路径 * @return true:存在,false:不存在 */ - boolean isFileExist(String filePath) throws IOException; + boolean isFileExist(String filePath); /** * 获取文件输入流 @@ -71,10 +54,18 @@ public interface IFtpHandler extends AutoCloseable { * * @param filePath 文件路径 * @param startPosition 指定的位置 - * @return input stream. + * @return */ InputStream getInputStreamByPosition(String filePath, long startPosition); + /** + * 列出指定路径下的目录 + * + * @param path 路径 + * @return 目录列表 + */ + List listDirs(String path); + /** * 列出指定路径下的目录 * @@ -89,7 +80,7 @@ public interface IFtpHandler extends AutoCloseable { * @param path 路径 * @return 文件列表 */ - List getFiles(String path) throws IOException; + List getFiles(String path); /** * 递归创建目录 @@ -112,14 +103,14 @@ public interface IFtpHandler extends AutoCloseable { * @param dir 指定的目录 * @param exclude 要排除的文件 */ - void deleteAllFilesInDir(String dir, List exclude) throws IOException; + void deleteAllFilesInDir(String dir, List exclude); /** * 删除文件 * * @param filePath 文件路径 */ - void deleteFile(String filePath) throws IOException; + boolean deleteFile(String filePath) throws IOException; /** * 重命名路径 diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/concurrent/ConcurrentFileSplit.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/concurrent/ConcurrentFileSplit.java new file mode 100644 index 0000000000..64c6aa9ef8 --- /dev/null +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/concurrent/ConcurrentFileSplit.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.connector.ftp.extend.ftp.concurrent; + +import com.dtstack.chunjun.connector.ftp.extend.ftp.IFormatConfig; +import com.dtstack.chunjun.connector.ftp.extend.ftp.IFtpHandler; + +import java.util.Comparator; +import java.util.List; + +public interface ConcurrentFileSplit { + + /** + * 切割文件， 生成 FtpFileSplit + * + * @param handler + * @param config + * @param files + * @return + */ + List buildFtpFileSplit( + IFtpHandler handler, IFormatConfig config, List files); + + /** + * 断点续传时候需要遍历FtpFileSplit，去除已经读过的文件, 所以需要保证FtpFileSplit的顺序; + * + * @return + */ + Comparator compare(); +} diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpFileSplit.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/concurrent/FtpFileSplit.java similarity index 93% rename from chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpFileSplit.java rename to chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/concurrent/FtpFileSplit.java index 8241a0305d..37bb29d1f8 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpFileSplit.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/concurrent/FtpFileSplit.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.dtstack.chunjun.connector.ftp.source; +package com.dtstack.chunjun.connector.ftp.extend.ftp.concurrent; import java.io.Serializable; import java.util.StringJoiner; @@ -26,16 +26,16 @@ public class FtpFileSplit implements Serializable { private static final long serialVersionUID = 1L; /** 这个分片处理文件的开始位置 */ - private long startPosition; + private long startPosition = 0; /** 这个分片处理文件的结束位置 */ - private long endPosition; + private long endPosition = 0; /** 这个分片处理的文件名 */ - private final String filename; + private String filename; /** 文件完整路径 */ - private final String fileAbsolutePath; + private String fileAbsolutePath; /** 压缩类型 */ private String compressType; diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/format/IFileReadFormat.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/format/IFileReadFormat.java similarity index 92% rename from chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/format/IFileReadFormat.java rename to chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/format/IFileReadFormat.java index 090c8b37a5..47355d3753 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/format/IFileReadFormat.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/extend/ftp/format/IFileReadFormat.java @@ -16,9 +16,10 @@ * limitations under the License. */ -package com.dtstack.chunjun.connector.ftp.format; +package com.dtstack.chunjun.connector.ftp.extend.ftp.format; -import com.dtstack.chunjun.connector.ftp.client.File; +import com.dtstack.chunjun.connector.ftp.extend.ftp.File; +import com.dtstack.chunjun.connector.ftp.extend.ftp.IFormatConfig; import java.io.Closeable; import java.io.IOException; diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/excel/ExcelReaderExceptionHandler.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/handler/DTFtpHandler.java similarity index 54% rename from chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/excel/ExcelReaderExceptionHandler.java rename to chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/handler/DTFtpHandler.java index c9f1124424..d6871ec900 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/client/excel/ExcelReaderExceptionHandler.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/handler/DTFtpHandler.java @@ -16,22 +16,33 @@ * limitations under the License. */ -package com.dtstack.chunjun.connector.ftp.client.excel; +package com.dtstack.chunjun.connector.ftp.handler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import com.dtstack.chunjun.connector.ftp.conf.FtpConfig; +import com.dtstack.chunjun.connector.ftp.extend.ftp.IFtpHandler; -/** @author by dujie @Description @Date 2021/12/20 */ -public class ExcelReaderExceptionHandler implements Thread.UncaughtExceptionHandler { +import java.io.IOException; - protected final Logger LOG = LoggerFactory.getLogger(getClass()); +public interface DTFtpHandler extends IFtpHandler { - @Override - public void uncaughtException(Thread t, Throwable e) { - LOG.error( - "an error occurred during the reading of the Excel file." - + " thread name : {}, message : {}", - t.getName(), - e.getMessage()); - } + /** + * 登录服务器 + * + * @param ftpConfig 连接配置 + */ + void loginFtpServer(FtpConfig ftpConfig); + + /** + * 登出服务器 + * + * @throws IOException logout error + */ + void logoutFtpServer() throws IOException; + + /** + * 关闭ftp输入流 + * + * @throws IOException 文件句柄操作异常 + */ + void completePendingCommand() throws IOException; } diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/handler/FtpHandler.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/handler/FtpHandler.java index adda70d0ae..63dacdac9a 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/handler/FtpHandler.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/handler/FtpHandler.java @@ -22,6 +22,7 @@ import com.dtstack.chunjun.connector.ftp.enums.EFtpMode; import com.dtstack.chunjun.constants.ConstantValue; import com.dtstack.chunjun.throwable.ChunJunRuntimeException; +import com.dtstack.chunjun.util.ExceptionUtil; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.io.IOUtils; @@ -40,11 +41,11 @@ import java.util.ArrayList; import java.util.List; -/** The concrete Ftp Utility class used for standard ftp */ -public class FtpHandler implements IFtpHandler { +public class FtpHandler implements DTFtpHandler { private static final Logger LOG = LoggerFactory.getLogger(FtpHandler.class); + private static final String DISCONNECT_FAIL_MESSAGE = "Failed to disconnect from ftp server"; private static final String SP = "/"; private FTPClient ftpClient = null; private String controlEncoding; @@ -150,7 +151,7 @@ public boolean isDirExist(String directoryPath) { } @Override - public boolean isFileExist(String filePath) throws IOException { + public boolean isFileExist(String filePath) { ftpClient.enterLocalPassiveMode(); InputStream inputStream = null; try { @@ -162,14 +163,19 @@ public boolean isFileExist(String filePath) throws IOException { + filePath); } finally { if (inputStream != null) { - inputStream.close(); - ftpClient.completePendingCommand(); + try { + inputStream.close(); + inputStream = null; + ftpClient.completePendingCommand(); + } catch (IOException e) { + throw new ChunJunRuntimeException("close input stream error."); + } } } } @Override - public List getFiles(String path) throws IOException { + public List getFiles(String path) { List sources = new ArrayList<>(); ftpClient.enterLocalPassiveMode(); @@ -183,7 +189,18 @@ public List getFiles(String path) throws IOException { path = path + SP; } try { - listFiles(path, sources); + FTPFile[] ftpFiles = ftpClient.listFiles(encodePath(path)); + if (ftpFiles != null) { + for (FTPFile ftpFile : ftpFiles) { + // .和..是特殊文件 + if (StringUtils.endsWith(ftpFile.getName(), ConstantValue.POINT_SYMBOL) + || StringUtils.endsWith( + ftpFile.getName(), ConstantValue.TWO_POINT_SYMBOL)) { + continue; + } + sources.addAll(getFiles(path + ftpFile.getName(), ftpFile)); + } + } } catch (IOException e) { LOG.error("", e); throw new RuntimeException(e); @@ -191,28 +208,13 @@ public List getFiles(String path) throws IOException { return sources; } - private void listFiles(String path, List sources) throws IOException { - FTPFile[] ftpFiles = ftpClient.listFiles(encodePath(path)); - if (ftpFiles != null) { - for (FTPFile ftpFile : ftpFiles) { - // .和..是特殊文件 - if (StringUtils.endsWith(ftpFile.getName(), ConstantValue.POINT_SYMBOL) - || StringUtils.endsWith( - ftpFile.getName(), ConstantValue.TWO_POINT_SYMBOL)) { - continue; - } - sources.addAll(getFiles(path + ftpFile.getName(), ftpFile)); - } - } - } - /** * 递归获取指定路径下的所有文件(暂无过滤) * - * @param path 指定路径 - * @param file FTP file - * @return 文件内容 - * @throws IOException io exception + * @param path + * @param file + * @return + * @throws IOException */ private List getFiles(String path, FTPFile file) throws IOException { List sources = new ArrayList<>(); @@ -221,7 +223,17 @@ private List getFiles(String path, FTPFile file) throws IOException { path = path + SP; } ftpClient.enterLocalPassiveMode(); - listFiles(path, sources); + FTPFile[] ftpFiles = ftpClient.listFiles(encodePath(path)); + if (ftpFiles != null) { + for (FTPFile ftpFile : ftpFiles) { + if (StringUtils.endsWith(ftpFile.getName(), ConstantValue.POINT_SYMBOL) + || StringUtils.endsWith( + ftpFile.getName(), ConstantValue.TWO_POINT_SYMBOL)) { + continue; + } + sources.addAll(getFiles(path + ftpFile.getName(), ftpFile)); + } + } } else { sources.add(path); } @@ -310,7 +322,7 @@ private void printWorkingDirectory() { } @Override - public void deleteAllFilesInDir(String dir, List exclude) throws IOException { + public void deleteAllFilesInDir(String dir, List exclude) { if (isDirExist(dir)) { if (!dir.endsWith(SP)) { dir = dir + SP; @@ -351,21 +363,23 @@ public void deleteAllFilesInDir(String dir, List exclude) throws IOExcep } @Override - public void deleteFile(String filePath) throws IOException { + public boolean deleteFile(String filePath) throws IOException { try { if (isFileExist(filePath)) { - ftpClient.deleteFile(filePath); + return ftpClient.deleteFile(filePath); } } catch (IOException e) { throw new IOException(e); } + return true; } @Override public InputStream getInputStream(String filePath) { try { ftpClient.enterLocalPassiveMode(); - return ftpClient.retrieveFileStream(encodePath(filePath)); + InputStream is = ftpClient.retrieveFileStream(encodePath(filePath)); + return is; } catch (IOException e) { String message = String.format("读取文件 : [%s] 时出错,请确认文件：[%s]存在且配置的用户有权限读取", filePath, filePath); @@ -376,20 +390,66 @@ public InputStream getInputStream(String filePath) { @Override public InputStream getInputStreamByPosition(String filePath, long startPosition) { - throw new RuntimeException("ftp协议，暂不支持从指定位置读取文件"); + if (startPosition != 0) { + throw new RuntimeException("ftp协议，暂不支持从指定位置读取文件"); + } + + return getInputStream(filePath); + } + + @Override + public List listDirs(String path) { + List sources = new ArrayList<>(); + if (isDirExist(path)) { + if (!path.endsWith(SP)) { + path = path + SP; + } + + try { + FTPFile[] ftpFiles = ftpClient.listFiles(encodePath(path)); + if (ftpFiles != null) { + for (FTPFile ftpFile : ftpFiles) { + if (StringUtils.endsWith(ftpFile.getName(), ConstantValue.POINT_SYMBOL) + || StringUtils.endsWith( + ftpFile.getName(), ConstantValue.TWO_POINT_SYMBOL)) { + continue; + } + sources.add(path + ftpFile.getName()); + } + } + } catch (IOException e) { + LOG.error("", e); + throw new RuntimeException(e); + } + } + + return sources; } @Override public void rename(String oldPath, String newPath) throws IOException { - /* 兼容windows, 如果目标文件存在, rename会报错 */ + /** 兼容windows, 如果目标文件存在, rename会报错 */ if (this.isFileExist(newPath)) { LOG.info(String.format("[%s] exist, delete it before rename", newPath)); - deleteFile(newPath); + this.deleteFile(newPath); } ftpClient.rename(encodePath(oldPath), encodePath(newPath)); } + @Override + public void completePendingCommand() throws IOException { + try { + // throw exception when return false + if (!ftpClient.completePendingCommand()) { + throw new IOException("I/O error occurs while sending or receiving data"); + } + } catch (IOException e) { + LOG.error("I/O error occurs while sending or receiving data"); + throw new IOException(ExceptionUtil.getErrorMessage(e)); + } + } + @Override public long getFileSize(String path) throws IOException { reconnectFtp(); @@ -443,6 +503,6 @@ public void reconnectFtp() { @Override public void close() throws Exception { - logoutFtpServer(); + this.logoutFtpServer(); } } diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/handler/FtpHandlerFactory.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/handler/FtpHandlerFactory.java index ad5e80323a..f6fe06cde7 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/handler/FtpHandlerFactory.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/handler/FtpHandlerFactory.java @@ -20,14 +20,10 @@ import org.apache.commons.lang3.StringUtils; -/** - * @author jiangbo - * @date 2019/11/21 - */ public class FtpHandlerFactory { - public static IFtpHandler createFtpHandler(String protocolStr) { - IFtpHandler ftpHandler; + public static DTFtpHandler createFtpHandler(String protocolStr) { + DTFtpHandler ftpHandler; Protocol protocol = Protocol.getByName(protocolStr); if (Protocol.SFTP.equals(protocol)) { diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/handler/Position.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/handler/Position.java index 4d1aa30b19..3c8ec0f41b 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/handler/Position.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/handler/Position.java @@ -18,7 +18,7 @@ package com.dtstack.chunjun.connector.ftp.handler; -import com.dtstack.chunjun.connector.ftp.source.FtpFileSplit; +import com.dtstack.chunjun.connector.ftp.extend.ftp.concurrent.FtpFileSplit; import java.io.Serializable; import java.util.StringJoiner; @@ -27,10 +27,10 @@ public class Position implements Serializable { private static final long serialVersionUID = 1L; - /** 第几条数据* */ - private final Long currentReadPosition; - /** 读取的数据文件* */ - private final FtpFileSplit fileSplit; + /** 当前的文件偏移量 */ + private Long currentReadPosition; + /** 读取的数据文件 */ + private FtpFileSplit fileSplit; public Position(Long currentReadPosition, FtpFileSplit fileSplit) { this.currentReadPosition = currentReadPosition; diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/handler/SftpHandler.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/handler/SftpHandler.java index 4082aeaf41..f233862d0e 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/handler/SftpHandler.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/handler/SftpHandler.java @@ -42,16 +42,15 @@ import java.util.Properties; import java.util.Vector; -/** The concrete Ftp Utility class used for sftp */ -public class SftpHandler implements IFtpHandler { +public class SftpHandler implements DTFtpHandler { private static final Logger LOG = LoggerFactory.getLogger(SftpHandler.class); private static final String DOT = "."; private static final String DOT_DOT = ".."; private static final String SP = "/"; private static final String SRC_MAIN = "src/main"; - private static final String[] PATH_NOT_EXIST_ERR = {"no such file", "is not a valid file path"}; - private static final String MSG_AUTH_FAIL = "Auth fail"; + private static String[] PATH_NOT_EXIST_ERR = {"no such file", "is not a valid file path"}; + private static String MSG_AUTH_FAIL = "Auth fail"; private Session session = null; private ChannelSftp channelSftp = null; @@ -225,6 +224,42 @@ public InputStream getInputStreamByPosition(String filePath, long startPosition) } } + @Override + public List listDirs(String path) { + if (StringUtils.isBlank(path)) { + path = SP; + } + + List dirs = new ArrayList<>(); + if (isDirExist(path)) { + if (path.equals(DOT) || path.equals(SRC_MAIN)) { + return dirs; + } + + if (!path.endsWith(SP)) { + path = path + SP; + } + + try { + Vector vector = channelSftp.ls(path); + for (int i = 0; i < vector.size(); ++i) { + ChannelSftp.LsEntry le = (ChannelSftp.LsEntry) vector.get(i); + String strName = le.getFilename(); + if (!strName.equals(DOT) + && !strName.equals(SRC_MAIN) + && !strName.equals(DOT_DOT)) { + String filePath = path + strName; + dirs.add(filePath); + } + } + } catch (SftpException e) { + LOG.error("", e); + } + } + + return dirs; + } + @Override public List getFiles(String path) { if (StringUtils.isBlank(path)) { @@ -239,9 +274,9 @@ public List getFiles(String path) { path = path + SP; } try { - Vector vector = channelSftp.ls(path); - for (Object o : vector) { - ChannelSftp.LsEntry le = (ChannelSftp.LsEntry) o; + Vector vector = channelSftp.ls(path); + for (int i = 0; i < vector.size(); ++i) { + ChannelSftp.LsEntry le = (ChannelSftp.LsEntry) vector.get(i); String strName = le.getFilename(); if (!strName.equals(DOT) && !strName.equals(SRC_MAIN) @@ -266,7 +301,7 @@ public List getFiles(String path) { public void mkDirRecursive(String directoryPath) { boolean isDirExist = false; try { - printWorkingDirectory(); + this.printWorkingDirectory(); SftpATTRS sftpAttrs = this.channelSftp.lstat(directoryPath); isDirExist = sftpAttrs.isDir(); } catch (SftpException e) { @@ -276,6 +311,7 @@ public void mkDirRecursive(String directoryPath) { String.format( "file not found, sftp server reply errorCode: [%d]", e.id); LOG.warn(message); + isDirExist = false; } } } @@ -338,9 +374,9 @@ public void deleteAllFilesInDir(String dir, List exclude) { } try { - Vector vector = channelSftp.ls(dir); - for (Object o : vector) { - ChannelSftp.LsEntry le = (ChannelSftp.LsEntry) o; + Vector vector = channelSftp.ls(dir); + for (int i = 0; i < vector.size(); ++i) { + ChannelSftp.LsEntry le = (ChannelSftp.LsEntry) vector.get(i); String strName = le.getFilename(); if (CollectionUtils.isNotEmpty(exclude) && exclude.contains(strName)) { continue; @@ -372,7 +408,7 @@ public void deleteAllFilesInDir(String dir, List exclude) { } @Override - public void deleteFile(String filePath) throws IOException { + public boolean deleteFile(String filePath) throws IOException { try { if (isFileExist(filePath)) { channelSftp.rm(filePath); @@ -380,36 +416,51 @@ public void deleteFile(String filePath) throws IOException { } catch (SftpException e) { throw new IOException(e); } + return true; } - public void mkDirSingleHierarchy(String directoryPath) throws SftpException { - boolean isDirExist; + public boolean mkDirSingleHierarchy(String directoryPath) throws SftpException { + boolean isDirExist = false; try { SftpATTRS sftpAttrs = this.channelSftp.lstat(directoryPath); isDirExist = sftpAttrs.isDir(); } catch (SftpException e) { - LOG.info("Creating a directory step by step [{}]", directoryPath); - this.channelSftp.mkdir(directoryPath); - return; + if (!isDirExist) { + LOG.info("Creating a directory step by step [{}]", directoryPath); + this.channelSftp.mkdir(directoryPath); + return true; + } } if (!isDirExist) { LOG.info("Creating a directory step by step [{}]", directoryPath); this.channelSftp.mkdir(directoryPath); } + return true; } @Override public void rename(String oldPath, String newPath) throws SftpException { + /** 兼容连通支付ftp数据源，如果目标文件存在会报错, 重命名前，先删除 */ + boolean deleteSuccess = true; if (this.isFileExist(newPath)) { try { LOG.info(String.format("[%s] exist, delete it before rename", newPath)); this.deleteFile(newPath); } catch (Exception e) { + deleteSuccess = false; throw new ChunJunRuntimeException(e); } } - channelSftp.rename(oldPath, newPath); + if (deleteSuccess) { + channelSftp.rename(oldPath, newPath); + } + } + + /** 仅ftp输入流需要显示关闭 */ + @Override + public void completePendingCommand() { + this.logoutFtpServer(); } @Override @@ -423,6 +474,6 @@ public long getFileSize(String path) throws IOException { @Override public void close() throws Exception { - logoutFtpServer(); + this.logoutFtpServer(); } } diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/format/CsvFileFormat.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/iformat/CsvFileFormat.java similarity index 91% rename from chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/format/CsvFileFormat.java rename to chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/iformat/CsvFileFormat.java index 594ac1f94b..a99013364e 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/format/CsvFileFormat.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/iformat/CsvFileFormat.java @@ -16,9 +16,11 @@ * limitations under the License. */ -package com.dtstack.chunjun.connector.ftp.format; +package com.dtstack.chunjun.connector.ftp.iformat; -import com.dtstack.chunjun.connector.ftp.client.File; +import com.dtstack.chunjun.connector.ftp.extend.ftp.File; +import com.dtstack.chunjun.connector.ftp.extend.ftp.IFormatConfig; +import com.dtstack.chunjun.connector.ftp.extend.ftp.format.IFileReadFormat; import com.csvreader.CsvReader; import org.apache.commons.collections.MapUtils; diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/format/ExcelFileFormat.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/iformat/ExcelFileFormat.java similarity index 91% rename from chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/format/ExcelFileFormat.java rename to chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/iformat/ExcelFileFormat.java index 3ee3b8286e..d69b204684 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/format/ExcelFileFormat.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/iformat/ExcelFileFormat.java @@ -16,13 +16,15 @@ * limitations under the License. */ -package com.dtstack.chunjun.connector.ftp.format; +package com.dtstack.chunjun.connector.ftp.iformat; -import com.dtstack.chunjun.connector.ftp.client.File; import com.dtstack.chunjun.connector.ftp.client.excel.ExcelReadListener; import com.dtstack.chunjun.connector.ftp.client.excel.ExcelReaderExecutor; import com.dtstack.chunjun.connector.ftp.client.excel.ExcelSubExceptionCarrier; import com.dtstack.chunjun.connector.ftp.client.excel.Row; +import com.dtstack.chunjun.connector.ftp.extend.ftp.File; +import com.dtstack.chunjun.connector.ftp.extend.ftp.IFormatConfig; +import com.dtstack.chunjun.connector.ftp.extend.ftp.format.IFileReadFormat; import com.alibaba.excel.EasyExcel; import com.alibaba.excel.ExcelReader; @@ -55,7 +57,7 @@ public class ExcelFileFormat implements IFileReadFormat { private Row row; @Override - public void open(File file, InputStream inputStream, IFormatConfig config) { + public void open(File file, InputStream inputStream, IFormatConfig config) throws IOException { LOG.info("open file : {}", file.getFileName()); this.cellCount = config.getFields().length; ExcelReadListener listener = new ExcelReadListener(); @@ -110,7 +112,7 @@ public boolean hasNext() throws IOException { } @Override - public String[] nextRecord() { + public String[] nextRecord() throws IOException { String[] data; if (row.isEnd()) { try { @@ -146,16 +148,16 @@ public void close() throws IOException { } private String[] formatValue(String[] data) { - String[] record = initDataContainer(cellCount); + String[] record = initDataContainer(cellCount, ""); // because cellCount is always >= data.length System.arraycopy(data, 0, record, 0, data.length); return record; } - private String[] initDataContainer(int capacity) { + private String[] initDataContainer(int capacity, String defValue) { String[] container = new String[capacity]; for (int i = 0; i < capacity; i++) { - container[i] = ""; + container[i] = defValue; } return container; } diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/format/IFormatFactory.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/iformat/IFormatFactory.java similarity index 91% rename from chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/format/IFormatFactory.java rename to chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/iformat/IFormatFactory.java index 69d9120128..0917e07fbe 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/format/IFormatFactory.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/iformat/IFormatFactory.java @@ -16,9 +16,10 @@ * limitations under the License. */ -package com.dtstack.chunjun.connector.ftp.format; +package com.dtstack.chunjun.connector.ftp.iformat; import com.dtstack.chunjun.connector.ftp.enums.FileType; +import com.dtstack.chunjun.connector.ftp.extend.ftp.format.IFileReadFormat; public class IFormatFactory { @@ -36,7 +37,7 @@ public static IFileReadFormat create(FileType fileType, String className) { } public static IFileReadFormat createCustom(String className) { - Class userClass; + Class userClass = null; try { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); userClass = classLoader.loadClass(className); diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/format/TextFileFormat.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/iformat/TextFileFormat.java similarity index 86% rename from chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/format/TextFileFormat.java rename to chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/iformat/TextFileFormat.java index fb22c4a674..86c12c8a5d 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/format/TextFileFormat.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/iformat/TextFileFormat.java @@ -16,9 +16,11 @@ * limitations under the License. */ -package com.dtstack.chunjun.connector.ftp.format; +package com.dtstack.chunjun.connector.ftp.iformat; -import com.dtstack.chunjun.connector.ftp.client.File; +import com.dtstack.chunjun.connector.ftp.extend.ftp.File; +import com.dtstack.chunjun.connector.ftp.extend.ftp.IFormatConfig; +import com.dtstack.chunjun.connector.ftp.extend.ftp.format.IFileReadFormat; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; @@ -52,7 +54,7 @@ public boolean hasNext() throws IOException { } @Override - public String[] nextRecord() { + public String[] nextRecord() throws IOException { return StringUtils.splitByWholeSeparatorPreserveAllTokens(line, filedDelimiter); } diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/options/FtpOptions.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/options/FtpOptions.java index 83d358c53d..dbd5d8c550 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/options/FtpOptions.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/options/FtpOptions.java @@ -23,11 +23,6 @@ import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; -/** - * @program chunjun - * @author: xiuzhu - * @create: 2021/06/19 - */ public class FtpOptions extends BaseFileOptions { public static final ConfigOption FORMAT = @@ -77,4 +72,16 @@ public class FtpOptions extends BaseFileOptions { .stringType() .defaultValue("PASV") .withDescription("ftp connectPattern"); + + public static final ConfigOption isFirstLineHeader = + ConfigOptions.key("isFirstLineHeader") + .booleanType() + .defaultValue(true) + .withDescription("FirstLineHeader"); + + public static final ConfigOption FILE_TYPE = + ConfigOptions.key("fileType") + .stringType() + .defaultValue("TXT") + .withDescription("FILE_TYPE"); } diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/sink/FtpDynamicTableSink.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/sink/FtpDynamicTableSink.java index 681af45477..4be0f775bc 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/sink/FtpDynamicTableSink.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/sink/FtpDynamicTableSink.java @@ -30,11 +30,6 @@ import org.apache.flink.table.connector.sink.SinkFunctionProvider; import org.apache.flink.table.data.RowData; -/** - * @program chunjun - * @author: xiuzhu - * @create: 2021/06/19 - */ public class FtpDynamicTableSink implements DynamicTableSink { private final TableSchema physicalSchema; diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/sink/FtpOutputFormat.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/sink/FtpOutputFormat.java index 5a8398e8c0..9317cc55c5 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/sink/FtpOutputFormat.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/sink/FtpOutputFormat.java @@ -19,8 +19,8 @@ package com.dtstack.chunjun.connector.ftp.sink; import com.dtstack.chunjun.connector.ftp.conf.FtpConfig; +import com.dtstack.chunjun.connector.ftp.handler.DTFtpHandler; import com.dtstack.chunjun.connector.ftp.handler.FtpHandlerFactory; -import com.dtstack.chunjun.connector.ftp.handler.IFtpHandler; import com.dtstack.chunjun.constants.ConstantValue; import com.dtstack.chunjun.enums.SizeUnitType; import com.dtstack.chunjun.sink.WriteMode; @@ -31,6 +31,8 @@ import org.apache.flink.table.data.RowData; +import org.apache.commons.lang3.StringUtils; + import java.io.BufferedWriter; import java.io.File; import java.io.IOException; @@ -39,8 +41,8 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; +import java.util.function.Function; -/** The OutputFormat Implementation which reads data from ftp servers. */ public class FtpOutputFormat extends BaseFileOutputFormat { /** 换行符 */ @@ -48,7 +50,7 @@ public class FtpOutputFormat extends BaseFileOutputFormat { protected FtpConfig ftpConfig; - private transient IFtpHandler ftpHandler; + private transient DTFtpHandler ftpHandler; private transient BufferedWriter writer; @@ -66,7 +68,7 @@ protected void openSource() {} @Override protected void checkOutputDir() { - wrapFtpHandler( + wrapFtpAction( iFtpHandler -> { if (iFtpHandler.isDirExist(tmpPath)) { if (iFtpHandler.isFileExist(tmpPath)) { @@ -76,17 +78,26 @@ protected void checkOutputDir() { } else { iFtpHandler.mkDirRecursive(tmpPath); } + return null; }); } @Override protected void deleteDataDir() { - wrapFtpHandler(iFtpHandler -> iFtpHandler.deleteAllFilesInDir(outputFilePath, null)); + wrapFtpAction( + iFtpHandler -> { + iFtpHandler.deleteAllFilesInDir(outputFilePath, null); + return null; + }); } @Override protected void deleteTmpDataDir() { - wrapFtpHandler(iFtpHandler -> iFtpHandler.deleteAllFilesInDir(tmpPath, null)); + wrapFtpAction( + iFtpHandler -> { + iFtpHandler.deleteAllFilesInDir(tmpPath, null); + return null; + }); } @Override @@ -105,6 +116,7 @@ protected void nextBlock() { } catch (IOException e) { throw new ChunJunRuntimeException(ExceptionUtil.getErrorMessage(e)); } + currentFileIndex++; } @@ -132,12 +144,12 @@ protected void checkCurrentFileSize() { } @Override - @SuppressWarnings("unchecked") public void writeSingleRecordToFile(RowData rowData) throws WriteRecordException { try { if (writer == null) { nextBlock(); } + String line = (String) rowConverter.toExternal(rowData, ""); this.writer.write(line); this.writer.write(NEWLINE); @@ -158,10 +170,15 @@ public void closeInternal() throws IOException { os.close(); os = null; } + this.ftpHandler.logoutFtpServer(); } catch (Exception e) { throw new ChunJunRuntimeException("can't close source.", e); } finally { - this.ftpHandler.logoutFtpServer(); + try { + this.ftpHandler.logoutFtpServer(); + } catch (IOException e) { + throw new ChunJunRuntimeException("can't logout ftp client.", e); + } } } @@ -237,8 +254,8 @@ protected void deleteDataFiles(List preCommitFilePathList, String path) @Override protected void moveAllTmpDataFileToDir() { - wrapFtpHandler( - handler -> { + wrapFtpAction( + (DTFtpHandler handler) -> { String dataFilePath = ""; try { List dataFiles = handler.getFiles(tmpPath); @@ -267,6 +284,7 @@ protected void moveAllTmpDataFileToDir() { dataFilePath, outputFilePath), e); } + return null; }); } @@ -292,7 +310,7 @@ protected long getCurrentFileSize() { @Override protected void writeMultipleRecordsInternal() { - throw new UnsupportedOperationException("FtpWriter don't support write batch data."); + notSupportBatchWrite("FtpWriter"); } public FtpConfig getFtpConfig() { @@ -303,23 +321,26 @@ public void setFtpConfig(FtpConfig ftpConfig) { this.ftpConfig = ftpConfig; } + protected void notSupportBatchWrite(String writerName) { + throw new UnsupportedOperationException(writerName + "不支持批量写入"); + } + private String handleUserSpecificFileName( - String tmpDataFileName, int fileNumber, List copyList, IFtpHandler handler) - throws IOException { + String tmpDataFileName, int fileNumber, List copyList, DTFtpHandler handler) { String fileName = ftpConfig.getFtpFileName(); - if (org.apache.commons.lang.StringUtils.isNotBlank(fileName)) { + if (StringUtils.isNotBlank(fileName)) { if (fileNumber == 1) { - fileName = handlerSingleFile(); + fileName = handlerSingleFile(tmpDataFileName); } else { fileName = handlerMultiChannel(tmpDataFileName); } } else { fileName = tmpDataFileName; } - return checkFilePath(fileName, copyList, handler); + return filepathCheck(fileName, copyList, handler); } - private String handlerSingleFile() { + private String handlerSingleFile(String tmpDataFileName) { return ftpConfig.getFtpFileName(); } @@ -344,8 +365,7 @@ private String handlerMultiChannel(String tmpDataFileName) { return fileName; } - private String checkFilePath(String filename, List copyList, IFtpHandler handler) - throws IOException { + private String filepathCheck(String filename, List copyList, DTFtpHandler handler) { if (WriteMode.NONCONFLICT.name().equalsIgnoreCase(ftpConfig.getWriteMode())) { if (handler.isFileExist(ftpConfig.getPath() + File.separatorChar + filename)) { handler.deleteAllFilesInDir(tmpPath, null); @@ -365,7 +385,7 @@ private String checkFilePath(String filename, List copyList, IFtpHandler } else if (WriteMode.INSERT.name().equalsIgnoreCase(ftpConfig.getWriteMode())) { if (handler.isFileExist(ftpConfig.getPath() + File.separatorChar + filename)) { String suffix = - org.apache.commons.lang.StringUtils.isNotBlank(ftpConfig.getSuffix()) + StringUtils.isNotBlank(ftpConfig.getSuffix()) ? "_" + ftpConfig.getSuffix() : "_" + UUID.randomUUID(); StringBuilder sb = new StringBuilder(filename); @@ -381,18 +401,20 @@ private String checkFilePath(String filename, List copyList, IFtpHandler return filename; } - private void wrapFtpHandler(Callback callback) { - try (IFtpHandler tmpFtpHandler = - FtpHandlerFactory.createFtpHandler(ftpConfig.getProtocol())) { - tmpFtpHandler.loginFtpServer(ftpConfig); - callback.apply(tmpFtpHandler); + private void wrapFtpAction(Function function) { + DTFtpHandler tmpFtpHandler = FtpHandlerFactory.createFtpHandler(ftpConfig.getProtocol()); + tmpFtpHandler.loginFtpServer(ftpConfig); + + try { + function.apply(tmpFtpHandler); } catch (Exception e) { throw new ChunJunRuntimeException(e); + } finally { + try { + tmpFtpHandler.logoutFtpServer(); + } catch (Exception e1) { + throw new ChunJunRuntimeException(e1); + } } } - - @FunctionalInterface - interface Callback { - void apply(T t) throws IOException; - } } diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/sink/FtpOutputFormatBuilder.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/sink/FtpOutputFormatBuilder.java index 8337bbebc3..d1b6bf829c 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/sink/FtpOutputFormatBuilder.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/sink/FtpOutputFormatBuilder.java @@ -24,14 +24,9 @@ import org.apache.commons.lang3.StringUtils; -/** - * The builder of FtpOutputFormat - * - *

Company: www.dtstack.com - * - * @author huyifan.zju@163.com - */ -public class FtpOutputFormatBuilder extends FileOutputFormatBuilder { +public class FtpOutputFormatBuilder extends FileOutputFormatBuilder { + + private FtpOutputFormat format; public FtpOutputFormatBuilder() { super(new FtpOutputFormat()); @@ -49,7 +44,7 @@ protected void checkFormat() { throw new ChunJunRuntimeException("Please Set protocol"); } if (StringUtils.isBlank(ftpConfig.getHost())) { - throw new ChunJunRuntimeException("Please Set gost"); + throw new ChunJunRuntimeException("Please Set host"); } if (StringUtils.isBlank(ftpConfig.getPath())) { throw new ChunJunRuntimeException("Please Set path"); diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/sink/FtpSinkFactory.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/sink/FtpSinkFactory.java index ed88c8adb8..a9d5748b0b 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/sink/FtpSinkFactory.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/sink/FtpSinkFactory.java @@ -38,11 +38,6 @@ import java.util.List; import java.util.stream.Collectors; -/** - * @program chunjun - * @author: xiuzhu - * @create: 2021/06/19 - */ public class FtpSinkFactory extends SinkFactory { private List columnName; diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpDynamicTableSource.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpDynamicTableSource.java index 5edaf84d38..2fb4f373db 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpDynamicTableSource.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpDynamicTableSource.java @@ -34,11 +34,6 @@ import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; import org.apache.flink.table.types.logical.RowType; -/** - * @program: Chunjun - * @author: xiuzhu - * @create: 2021/06/19 - */ public class FtpDynamicTableSource implements ScanTableSource { private TableSchema schema; diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpFileReader.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpFileReader.java index 8e2adc0d51..2fc28036b2 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpFileReader.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpFileReader.java @@ -19,16 +19,23 @@ package com.dtstack.chunjun.connector.ftp.source; import com.dtstack.chunjun.connector.ftp.client.Data; -import com.dtstack.chunjun.connector.ftp.client.File; -import com.dtstack.chunjun.connector.ftp.client.FileUtil; import com.dtstack.chunjun.connector.ftp.client.ZipInputStream; import com.dtstack.chunjun.connector.ftp.conf.FtpConfig; import com.dtstack.chunjun.connector.ftp.enums.FileType; -import com.dtstack.chunjun.connector.ftp.format.IFileReadFormat; -import com.dtstack.chunjun.connector.ftp.format.IFormatConfig; -import com.dtstack.chunjun.connector.ftp.format.IFormatFactory; -import com.dtstack.chunjun.connector.ftp.handler.IFtpHandler; +import com.dtstack.chunjun.connector.ftp.extend.ftp.File; +import com.dtstack.chunjun.connector.ftp.extend.ftp.FtpParseException; +import com.dtstack.chunjun.connector.ftp.extend.ftp.IFormatConfig; +import com.dtstack.chunjun.connector.ftp.extend.ftp.concurrent.FtpFileSplit; +import com.dtstack.chunjun.connector.ftp.extend.ftp.format.IFileReadFormat; +import com.dtstack.chunjun.connector.ftp.handler.DTFtpHandler; +import com.dtstack.chunjun.connector.ftp.handler.FtpHandler; +import com.dtstack.chunjun.connector.ftp.handler.FtpHandlerFactory; import com.dtstack.chunjun.connector.ftp.handler.Position; +import com.dtstack.chunjun.connector.ftp.iformat.IFormatFactory; +import com.dtstack.chunjun.metrics.BaseMetric; + +import org.apache.flink.api.common.accumulators.LongCounter; +import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -36,8 +43,6 @@ import java.io.IOException; import java.io.InputStream; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -45,27 +50,31 @@ import java.util.Locale; import java.util.Map; +import static com.dtstack.chunjun.connector.ftp.conf.ConfigConstants.FTP_COUNTER_PREFIX; + public class FtpFileReader { private static final Logger LOG = LoggerFactory.getLogger(FtpFileReader.class); - private final IFtpHandler ftpHandler; + private DTFtpHandler ftpHandler; private Iterator iter; private int fromLine = 0; /** the fileSplit current read * */ private FtpFileSplit currentFileSplit; - /** The bytes read from the current fileSplit * */ - private Long currentFileSplitReadBytes = 0L; - private Position startPosition; private final FtpConfig ftpConfig; private IFormatConfig iFormatConfig; private IFileReadFormat currentFileReadFormat; private final Map iFileReadFormatCache; + private RuntimeContext runtimeContext; + private Map counterMap; + private BaseMetric baseMetric; + private boolean enableMetric = false; + private FtpInputStreamProxy currentInputStream; public FtpFileReader( - IFtpHandler ftpHandler, + DTFtpHandler ftpHandler, Iterator iter, FtpConfig ftpConfig, Position startPosition) { @@ -79,20 +88,19 @@ public FtpFileReader( /** 断点续跑，过滤文件已经读过的部分 * */ public void skipHasReadFiles() { if (startPosition != null && startPosition.getCurrentReadPosition() > 0) { - FtpFileSplit storeFileSplit = startPosition.getFileSplit(); + FtpFileSplit storefs = startPosition.getFileSplit(); - /* + /** * remove same file name but endPosition < startPosition.getCurrentReadPosition() set * FtpFileSplit startPosition = startPosition.getCurrentReadPosition() */ ArrayList fileCache = new ArrayList<>(); for (Iterator it = iter; it.hasNext(); ) { FtpFileSplit fs = it.next(); - if (fs.getFileAbsolutePath().equals(storeFileSplit.getFileAbsolutePath())) { + if (fs.getFileAbsolutePath().equals(storefs.getFileAbsolutePath())) { if (fs.getEndPosition() <= startPosition.getCurrentReadPosition()) { // remove same file name but endPosition < // startPosition.getCurrentReadPosition() - // do nothing } else { fs.setStartPosition(startPosition.getCurrentReadPosition()); fileCache.add(fs); @@ -112,23 +120,50 @@ public Data readLine() throws IOException { } if (currentFileReadFormat != null) { - if (currentFileSplitReadBytes >= currentFileSplit.getReadLimit()) { - close(); - return readLine(); - } - if (!currentFileReadFormat.hasNext()) { close(); return readLine(); } - String[] record = currentFileReadFormat.nextRecord(); - addCurrentReadSize(record); + try { + String[] record = currentFileReadFormat.nextRecord(); + if (enableMetric) { + String readBytesMetricName = getMetricName("readBytes"); + LongCounter readBytesCounter = counterMap.get(readBytesMetricName); + setMetricValue(readBytesCounter, currentInputStream.getCurrentReadBytes()); - return new Data(record, new Position(currentFileSplitReadBytes, currentFileSplit)); - } + String metricName = getMetricName("readLines"); + LongCounter counter = counterMap.get(metricName); + updateMetric(counter, 1); + } + + return new Data( + record, + new Position( + currentInputStream.getCurrentReadBytes() + + currentFileSplit.getStartPosition(), + currentFileSplit)); + + } catch (FtpParseException e) { + String[] record = new String[] {e.getContent()}; + return new Data( + record, + new Position( + currentInputStream.getCurrentReadBytes() + + currentFileSplit.getStartPosition(), + currentFileSplit), + e); + } - return null; + } else { + if (enableMetric) { + // tick end + String metricName = getMetricName("tickEnd"); + LongCounter counter = counterMap.get(metricName); + tickEndMetric(counter); + } + return null; + } } private void nextStream() throws IOException { @@ -164,6 +199,8 @@ private void nextStream() throws IOException { iFileReadFormatCache.put(fileType, iFileReadFormat); } currentFileReadFormat = iFileReadFormatCache.get(fileType); + currentFileSplit = fileSplit; + currentInputStream = new FtpInputStreamProxy(in, currentFileSplit.getReadLimit()); // adapt to previous file parameter File file = @@ -172,10 +209,7 @@ private void nextStream() throws IOException { fileSplit.getFileAbsolutePath(), fileSplit.getFilename(), fileSplit.getCompressType()); - currentFileReadFormat.open(file, in, iFormatConfig); - - currentFileSplit = fileSplit; - currentFileSplitReadBytes = 0L; + currentFileReadFormat.open(file, currentInputStream, iFormatConfig); if (fileSplit.getStartPosition() == 0) { if (fileType != FileType.EXCEL) { @@ -183,7 +217,6 @@ private void nextStream() throws IOException { if (currentFileReadFormat.hasNext()) { String[] strings = currentFileReadFormat.nextRecord(); LOG.info("Skip line:{}", Arrays.toString(strings)); - addCurrentReadSize(strings); } else { break; } @@ -196,11 +229,30 @@ private void nextStream() throws IOException { } public void close() throws IOException { + + if (currentInputStream != null) { + currentInputStream.close(); + } + if (currentFileReadFormat != null) { currentFileReadFormat.close(); currentFileReadFormat = null; - FileUtil.closeWithFtpHandler(ftpHandler, LOG); + if (ftpHandler instanceof FtpHandler) { + try { + ((FtpHandler) ftpHandler).getFtpClient().completePendingCommand(); + } catch (Exception e) { + // 如果出现了超时异常，就直接获取一个新的ftpHandler + LOG.warn("FTPClient completePendingCommand has error ->", e); + try { + ftpHandler.logoutFtpServer(); + } catch (Exception exception) { + LOG.warn("FTPClient logout has error ->", exception); + } + ftpHandler = FtpHandlerFactory.createFtpHandler(ftpConfig.getProtocol()); + ftpHandler.loginFtpServer(ftpConfig); + } + } } } @@ -243,23 +295,102 @@ public String getCurrentFileName() { return currentFileSplit.getFileAbsolutePath(); } - public void setIFormatConfig(IFormatConfig iFormatConfig) { + public IFormatConfig getiFormatConfig() { + return iFormatConfig; + } + + public void setiFormatConfig(IFormatConfig iFormatConfig) { this.iFormatConfig = iFormatConfig; } - private void addCurrentReadSize(String[] value) { - String line = String.join(ftpConfig.getFieldDelimiter(), value); - currentFileSplitReadBytes += line.getBytes(getCharacterSet()).length; - currentFileSplitReadBytes += "\n".getBytes(getCharacterSet()).length; + public void enableMetric(RuntimeContext runtimeContext, BaseMetric baseMetric) { + this.runtimeContext = runtimeContext; + this.baseMetric = baseMetric; + counterMap = new HashMap<>(); + this.enableMetric = true; + + ArrayList fileCache = new ArrayList<>(); + + int totalFiles = 0; + StringBuilder allFileNamesBuilder = new StringBuilder(); + allFileNamesBuilder.append("ftp_read_files("); + + for (Iterator it = iter; it.hasNext(); ) { + FtpFileSplit file = it.next(); + String readLinesMetricName = getMetricName("readLines"); + String readBytesMetricName = getMetricName("readBytes"); + + counterMap.put(readLinesMetricName, registerMetric(readLinesMetricName, true)); + counterMap.put(readBytesMetricName, registerMetric(readBytesMetricName, false)); + + totalFiles += 1; + allFileNamesBuilder.append(file.getFileAbsolutePath()); + + if (it.hasNext()) { + allFileNamesBuilder.append(", "); + } + fileCache.add(file); + } + + allFileNamesBuilder.append(')'); + String totalFileMetricName = allFileNamesBuilder.toString(); + LongCounter totalFileCounter = registerMetric(totalFileMetricName, false); + counterMap.put(totalFileMetricName, registerMetric(totalFileMetricName, false)); + updateMetric(totalFileCounter, totalFiles); + + // tick start + String tickStartMetric = getMetricName("tickStart"); + LongCounter durationCounter = registerMetric(tickStartMetric, false); + counterMap.put(tickStartMetric, durationCounter); + tickStartMetric(durationCounter); + + iter = fileCache.iterator(); } - private Charset getCharacterSet() { - switch (ftpConfig.encoding) { - case "gbk": - return Charset.forName("GBK"); - case "utf-8": + private String getMetricName(String action) { + switch (action) { + case "tickStart": + case "tickEnd": + return FTP_COUNTER_PREFIX + "_read_duration"; + + case "readLines": + return FTP_COUNTER_PREFIX + "_read_lines"; + + case "readBytes": + return FTP_COUNTER_PREFIX + "_read_bytes"; + default: - return StandardCharsets.UTF_8; + throw new RuntimeException("illegal parameter"); + } + } + + public LongCounter registerMetric(String metricName, boolean meterView) { + LongCounter counter = runtimeContext.getLongCounter(metricName); + baseMetric.addMetric(metricName, counter, meterView); + return counter; + } + + public void updateMetric(LongCounter counter, long incr) { + counter.add(incr); + } + + public void setMetricValue(LongCounter counter, Long value) { + counter.resetLocal(); + counter.add(value); + } + + public void tickStartMetric(LongCounter counter) { + if (counter != null) { + counter.resetLocal(); + counter.add(System.currentTimeMillis()); + } + } + + public void tickEndMetric(LongCounter counter) { + if (counter != null) { + Long startTime = counter.getLocalValue(); + counter.resetLocal(); + counter.add(System.currentTimeMillis() - startTime); } } } diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpInputFormat.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpInputFormat.java index e4e3e9e94e..b888951a46 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpInputFormat.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpInputFormat.java @@ -20,15 +20,17 @@ import com.dtstack.chunjun.conf.FieldConf; import com.dtstack.chunjun.connector.ftp.client.Data; -import com.dtstack.chunjun.connector.ftp.client.FileUtil; import com.dtstack.chunjun.connector.ftp.conf.ConfigConstants; import com.dtstack.chunjun.connector.ftp.conf.FtpConfig; import com.dtstack.chunjun.connector.ftp.converter.FtpColumnConverter; import com.dtstack.chunjun.connector.ftp.converter.FtpRowConverter; -import com.dtstack.chunjun.connector.ftp.format.IFormatConfig; +import com.dtstack.chunjun.connector.ftp.extend.ftp.IFormatConfig; +import com.dtstack.chunjun.connector.ftp.extend.ftp.concurrent.ConcurrentFileSplit; +import com.dtstack.chunjun.connector.ftp.extend.ftp.concurrent.FtpFileSplit; +import com.dtstack.chunjun.connector.ftp.handler.DTFtpHandler; import com.dtstack.chunjun.connector.ftp.handler.FtpHandlerFactory; -import com.dtstack.chunjun.connector.ftp.handler.IFtpHandler; import com.dtstack.chunjun.connector.ftp.handler.Position; +import com.dtstack.chunjun.connector.ftp.spliter.ConcurrentFileSplitFactory; import com.dtstack.chunjun.constants.ConstantValue; import com.dtstack.chunjun.restore.FormatState; import com.dtstack.chunjun.source.format.BaseRichInputFormat; @@ -41,12 +43,11 @@ import org.apache.flink.table.data.RowData; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -65,7 +66,7 @@ public class FtpInputFormat extends BaseRichInputFormat { private transient FtpFileReader reader; - private transient IFtpHandler ftpHandler; + private transient DTFtpHandler ftpHandler; private transient Data data; @@ -86,7 +87,7 @@ public void openInputFormat() throws IOException { @Override public InputSplit[] createInputSplitsInternal(int minNumSplits) throws Exception { - IFtpHandler ftpHandler = FtpHandlerFactory.createFtpHandler(ftpConfig.getProtocol()); + DTFtpHandler ftpHandler = FtpHandlerFactory.createFtpHandler(ftpConfig.getProtocol()); ftpHandler.loginFtpServer(ftpConfig); List files = new ArrayList<>(); @@ -101,39 +102,19 @@ public InputSplit[] createInputSplitsInternal(int minNumSplits) throws Exception } } - List fileList = new ArrayList<>(); + ConcurrentFileSplit spliter = + ConcurrentFileSplitFactory.createConcurrentFileSplit(ftpConfig); + List fileList = + spliter.buildFtpFileSplit(ftpHandler, buildIFormatConfig(ftpConfig), files); - if (CollectionUtils.isNotEmpty(files)) { - for (String filePath : files) { - // add file with compressType - if (org.apache.commons.lang.StringUtils.isNotBlank( - ftpConfig.getCompressType())) { - FileUtil.addCompressFile(ftpHandler, filePath, ftpConfig, fileList); - - } else { - // add normal file - FileUtil.addFile(ftpHandler, filePath, ftpConfig, fileList); - } - } - } - if (CollectionUtils.isEmpty(fileList)) { - throw new RuntimeException("There are no readable files in directory " + path); - } - - FtpInputSplit[] ftpInputSplits = new FtpInputSplit[minNumSplits]; - for (int index = 0; index < minNumSplits; ++index) { + int numSplits = minNumSplits; + FtpInputSplit[] ftpInputSplits = new FtpInputSplit[numSplits]; + for (int index = 0; index < numSplits; ++index) { ftpInputSplits[index] = new FtpInputSplit(); } - // 先根据文件路径排序 - // 再根据文件里面开始的偏移量排序, 从小到大 - fileList.sort( - Comparator.comparing(FtpFileSplit::getFileAbsolutePath) - .thenComparing( - FtpFileSplit::getStartPosition, Comparator.naturalOrder())); - for (int i = 0; i < fileList.size(); ++i) { - ftpInputSplits[i % minNumSplits].getFileSplits().add(fileList.get(i)); + ftpInputSplits[i % numSplits].getFileSplits().add(fileList.get(i)); } return ftpInputSplits; } finally { @@ -142,7 +123,7 @@ public InputSplit[] createInputSplitsInternal(int minNumSplits) throws Exception } @Override - public void openInternal(InputSplit split) { + public void openInternal(InputSplit split) throws IOException { ftpHandler = FtpHandlerFactory.createFtpHandler(ftpConfig.getProtocol()); ftpHandler.loginFtpServer(ftpConfig); @@ -163,7 +144,8 @@ public void openInternal(InputSplit split) { reader.setFromLine(0); } - reader.setIFormatConfig(buildIFormatConfig(ftpConfig)); + reader.setiFormatConfig(buildIFormatConfig(ftpConfig)); + reader.enableMetric(getRuntimeContext(), inputMetric); reader.skipHasReadFiles(); } @@ -174,12 +156,15 @@ public boolean reachedEnd() throws IOException { } @Override - @SuppressWarnings("unchecked") protected RowData nextRecordInternal(RowData rowData) throws ReadRecordException { String[] fields = data.getData(); + if (data.getException() != null) { + throw new ReadRecordException( + data.getException().getMessage(), data.getException(), 0, rowData); + } try { - if (fields.length == 1 && org.apache.commons.lang.StringUtils.isBlank(fields[0])) { + if (fields.length == 1 && StringUtils.isBlank(fields[0])) { LOG.warn("read data:{}, it will not be written.", Arrays.toString(fields)); return null; } @@ -220,7 +205,7 @@ protected RowData nextRecordInternal(RowData rowData) throws ReadRecordException for (int i = 0; i < CollectionUtils.size(columns); i++) { FieldConf fieldConf = columns.get(i); - Object value; + Object value = null; if (fieldConf.getValue() != null) { value = fieldConf.getValue(); } else { @@ -243,7 +228,7 @@ protected RowData nextRecordInternal(RowData rowData) throws ReadRecordException rowData = rowConverter.toInternal(genericRowData); } } catch (Exception e) { - throw new ReadRecordException("Read data error.", e, 0, fields); + throw new ReadRecordException("Read data error.", e, 0, rowData); } position = data.getPosition(); return rowData; @@ -284,12 +269,19 @@ private IFormatConfig buildIFormatConfig(FtpConfig ftpConfig) { iFormatConfig.setFileConfig(ftpConfig.getFileConfig()); final String[] fields = new String[ftpConfig.getColumn().size()]; IntStream.range(0, fields.length) - .forEach(i -> fields[i] = ftpConfig.getColumn().get(i).getName()); + .forEach( + i -> { + fields[i] = ftpConfig.getColumn().get(i).getName(); + }); iFormatConfig.setFields(fields); + iFormatConfig.setFetchMaxSize(ftpConfig.getMaxFetchSize()); + iFormatConfig.setParallelism(ftpConfig.getParallelism()); + iFormatConfig.setColumnDelimiter(ftpConfig.getColumnDelimiter()); + return iFormatConfig; } - private List listFilesInPath(IFtpHandler ftpHandler, String path) throws IOException { + private List listFilesInPath(DTFtpHandler ftpHandler, String path) { path = path.trim(); String fileRegex = path.substring(path.lastIndexOf("/") + 1); boolean isRegex = StringUtils.containsAny(fileRegex, REGEX_CHARS); @@ -343,12 +335,13 @@ public void closeInputFormat() { Map allCounters = inputMetric.getMetricCounters(); Map ftpCounter = new HashMap<>(); - allCounters.forEach( - (key, value) -> { - if (key.startsWith(FTP_COUNTER_PREFIX)) { - ftpCounter.put(key, value.getLocalValue()); - } - }); + allCounters.entrySet().stream() + .forEach( + kv -> { + if (kv.getKey().startsWith(FTP_COUNTER_PREFIX)) { + ftpCounter.put(kv.getKey(), kv.getValue().getLocalValue()); + } + }); PrintUtil.printResult(ftpCounter); diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpInputFormatBuilder.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpInputFormatBuilder.java index de73b8dc40..5c299cf3e3 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpInputFormatBuilder.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpInputFormatBuilder.java @@ -24,7 +24,6 @@ import org.apache.commons.lang3.StringUtils; -/** @author jiangbo */ public class FtpInputFormatBuilder extends BaseRichInputFormatBuilder { public FtpInputFormatBuilder() { diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpInputSplit.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpInputSplit.java index 280782b3d0..f047c7bcce 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpInputSplit.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpInputSplit.java @@ -18,6 +18,8 @@ package com.dtstack.chunjun.connector.ftp.source; +import com.dtstack.chunjun.connector.ftp.extend.ftp.concurrent.FtpFileSplit; + import org.apache.flink.core.io.InputSplit; import java.util.ArrayList; diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpInputStreamProxy.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpInputStreamProxy.java new file mode 100644 index 0000000000..fd4a2918eb --- /dev/null +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpInputStreamProxy.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.connector.ftp.source; + +import java.io.IOException; +import java.io.InputStream; + +public class FtpInputStreamProxy extends InputStream { + + private InputStream input; + private long readLimit; + private long currentReadBytes; + + public FtpInputStreamProxy(InputStream input, long readLimit) { + this.input = input; + this.readLimit = readLimit; + this.currentReadBytes = 0; + } + + @Override + public int read() throws IOException { + if (currentReadBytes + 1 > readLimit) { + return -1; + } else { + int ch = input.read(); + currentReadBytes += 1; + return ch; + } + } + + @Override + public int read(byte[] b) throws IOException { + return read(b, 0, b.length); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (currentReadBytes + len <= readLimit) { + int ch = input.read(b, off, len); + currentReadBytes += ch; + return ch; + } else { + int maxRead = (int) (readLimit - currentReadBytes); + if (maxRead > 0) { + int ch = input.read(b, off, maxRead); + currentReadBytes += ch; + return ch; + } else { + return -1; + } + } + } + + @Override + public long skip(long n) throws IOException { + return input.skip(n); + } + + @Override + public synchronized void reset() throws IOException { + input.reset(); + } + + @Override + public boolean markSupported() { + return input.markSupported(); + } + + @Override + public int available() throws IOException { + if (currentReadBytes > readLimit) { + return -1; + } + + return input.available(); + } + + @Override + public void close() throws IOException { + input.close(); + } + + @Override + public synchronized void mark(int readlimit) { + input.mark(readlimit); + } + + public long getCurrentReadBytes() { + return currentReadBytes; + } +} diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpSourceFactory.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpSourceFactory.java index 11b8b985c2..d90e1b9efb 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpSourceFactory.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/source/FtpSourceFactory.java @@ -18,7 +18,6 @@ package com.dtstack.chunjun.connector.ftp.source; -import com.dtstack.chunjun.conf.FieldConf; import com.dtstack.chunjun.conf.SyncConf; import com.dtstack.chunjun.connector.ftp.conf.ConfigConstants; import com.dtstack.chunjun.connector.ftp.conf.FtpConfig; @@ -35,17 +34,9 @@ import org.apache.flink.table.data.RowData; import org.apache.flink.table.types.logical.RowType; -import java.util.List; -import java.util.stream.Collectors; - -/** - * @program chunjun - * @author: xiuzhu - * @create: 2021/06/19 - */ public class FtpSourceFactory extends SourceFactory { - private final FtpConfig ftpConfig; + private FtpConfig ftpConfig; public FtpSourceFactory(SyncConf syncConf, StreamExecutionEnvironment env) { super(syncConf, env); @@ -61,6 +52,8 @@ public FtpSourceFactory(SyncConf syncConf, StreamExecutionEnvironment env) { String fieldDelimiter = StringUtil.convertRegularExpr(ftpConfig.getFieldDelimiter()); ftpConfig.setFieldDelimiter(fieldDelimiter); } + + ftpConfig.setColumn(syncConf.getReader().getFieldList()); super.initCommonConf(ftpConfig); } @@ -68,16 +61,6 @@ public FtpSourceFactory(SyncConf syncConf, StreamExecutionEnvironment env) { public DataStream createSource() { FtpInputFormatBuilder builder = new FtpInputFormatBuilder(); builder.setFtpConfig(ftpConfig); - List fieldConfList = - ftpConfig.getColumn().stream() - .peek( - fieldConf -> { - if (fieldConf.getName() == null) { - fieldConf.setName(String.valueOf(fieldConf.getIndex())); - } - }) - .collect(Collectors.toList()); - ftpConfig.setColumn(fieldConfList); final RowType rowType = TableUtil.createRowType(ftpConfig.getColumn(), getRawTypeConverter()); builder.setRowConverter(new FtpColumnConverter(rowType, ftpConfig), useAbstractBaseColumn); diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/spliter/ConcurrentCsvSplit.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/spliter/ConcurrentCsvSplit.java new file mode 100644 index 0000000000..105ad39e94 --- /dev/null +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/spliter/ConcurrentCsvSplit.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.connector.ftp.spliter; + +import com.dtstack.chunjun.connector.ftp.client.FileUtil; +import com.dtstack.chunjun.connector.ftp.extend.ftp.IFormatConfig; +import com.dtstack.chunjun.connector.ftp.extend.ftp.IFtpHandler; +import com.dtstack.chunjun.connector.ftp.extend.ftp.concurrent.FtpFileSplit; +import com.dtstack.chunjun.throwable.ChunJunRuntimeException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +public class ConcurrentCsvSplit extends DefaultFileSplit { + private static final Logger LOG = LoggerFactory.getLogger(ConcurrentCsvSplit.class); + + @Override + public List buildFtpFileSplit( + IFtpHandler handler, IFormatConfig config, List files) { + List fileSplits = new ArrayList<>(); + + for (String filePath : files) { + fileSplits.addAll(analyseSingleFile(handler, config, filePath)); + } + return fileSplits; + } + + public List analyseSingleFile( + IFtpHandler ftpHandler, IFormatConfig config, String filePath) { + List ftpFileSplits = new ArrayList<>(); + + String columnDelimiter = config.getColumnDelimiter(); + try { + long currentFileSize = ftpHandler.getFileSize(filePath); + long fetchSize = Math.max(1024 * 1024, config.getFetchMaxSize()); + String filename = FileUtil.getFilename(filePath); + if (currentFileSize > fetchSize) { + int perSplit = + Math.min( + (int) currentFileSize / config.getParallelism(), Integer.MAX_VALUE); + long startPosition = 0; + long endPosition = startPosition + perSplit; + + while (endPosition <= currentFileSize) { + if (endPosition == currentFileSize) { + FtpFileSplit ftpFileSplit = + new FtpFileSplit(startPosition, endPosition, filePath, filename); + ftpFileSplits.add(ftpFileSplit); + break; + } + + InputStream input = ftpHandler.getInputStreamByPosition(filePath, endPosition); + boolean notMatch = true; + + while (notMatch) { + notMatch = false; + for (char c : columnDelimiter.toCharArray()) { + char ch = (char) input.read(); + endPosition += 1; + + if (ch != c) { + notMatch = true; + break; + } + } + } + + FtpFileSplit ftpFileSplit = + new FtpFileSplit(startPosition, endPosition, filePath, filename); + ftpFileSplits.add(ftpFileSplit); + + LOG.info( + String.format( + "build file split, filename: %s, startPosition: %d, endPosition: %d", + filePath, startPosition, endPosition)); + + startPosition = endPosition; + endPosition = startPosition + perSplit; + } + + if (endPosition != currentFileSize) { + FtpFileSplit ftpFileSplit = + new FtpFileSplit(startPosition, currentFileSize, filePath, filename); + ftpFileSplits.add(ftpFileSplit); + + LOG.info( + String.format( + "build file split, filename: %s, startPosition: %d, endPosition: %d", + filePath, startPosition, currentFileSize)); + } + } else { + ftpFileSplits.add(new FtpFileSplit(0, currentFileSize, filePath, filename)); + } + + ftpFileSplits.sort(compare()); + + return ftpFileSplits; + } catch (Exception e) { + throw new ChunJunRuntimeException(e); + } + } +} diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/spliter/ConcurrentFileSplitFactory.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/spliter/ConcurrentFileSplitFactory.java new file mode 100644 index 0000000000..ee5bb16f76 --- /dev/null +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/spliter/ConcurrentFileSplitFactory.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.connector.ftp.spliter; + +import com.dtstack.chunjun.connector.ftp.conf.FtpConfig; +import com.dtstack.chunjun.connector.ftp.extend.ftp.concurrent.ConcurrentFileSplit; +import com.dtstack.chunjun.throwable.ChunJunRuntimeException; + +import org.apache.commons.lang3.StringUtils; + +import java.lang.reflect.Constructor; +import java.util.Locale; + +public class ConcurrentFileSplitFactory { + + public static ConcurrentFileSplit createConcurrentFileSplit(FtpConfig config) { + /** user defined */ + String customSplitClassName = config.getCustomConcurrentFileSplitClassName(); + if (StringUtils.isNotBlank(customSplitClassName)) { + try { + Class clazz = Class.forName(customSplitClassName); + Constructor constructor = clazz.getConstructor(); + return (ConcurrentFileSplit) constructor.newInstance(); + } catch (Exception e) { + throw new ChunJunRuntimeException(e); + } + } + + /** compress file */ + String compressType = config.getCompressType(); + if (StringUtils.isNotBlank(compressType)) { + if (compressType.toUpperCase(Locale.ENGLISH).equals("ZIP")) { + return new ConcurrentZipCompressSplit(); + } else { + throw new ChunJunRuntimeException("not support compress type"); + } + } + + /** normal file, csv, excel, txt */ + String fileType = config.getFileType(); + switch (fileType.toUpperCase(Locale.ENGLISH)) { + case "CSV": + case "TXT": + return new ConcurrentCsvSplit(); + + case "EXCEL": + default: + return new DefaultFileSplit(); + } + } +} diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/spliter/ConcurrentZipCompressSplit.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/spliter/ConcurrentZipCompressSplit.java new file mode 100644 index 0000000000..3a9c5b67cc --- /dev/null +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/spliter/ConcurrentZipCompressSplit.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.connector.ftp.spliter; + +import com.dtstack.chunjun.connector.ftp.extend.ftp.IFormatConfig; +import com.dtstack.chunjun.connector.ftp.extend.ftp.IFtpHandler; +import com.dtstack.chunjun.connector.ftp.extend.ftp.concurrent.FtpFileSplit; +import com.dtstack.chunjun.throwable.ChunJunRuntimeException; + +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +public class ConcurrentZipCompressSplit extends DefaultFileSplit { + @Override + public List buildFtpFileSplit( + IFtpHandler handler, IFormatConfig config, List files) { + List fileSplits = new ArrayList<>(); + for (String filePath : files) { + fileSplits.addAll(analyseSingleFile(handler, filePath, config.getEncoding(), "ZIP")); + } + + fileSplits.sort(compare()); + return fileSplits; + } + + private List analyseSingleFile( + IFtpHandler ftpHandler, String filePath, String encoding, String compressType) { + + List fileSplits = new ArrayList<>(); + try (java.util.zip.ZipInputStream zipInputStream = + new ZipInputStream( + ftpHandler.getInputStream(filePath), Charset.forName(encoding))) { + ZipEntry zipEntry; + + while ((zipEntry = zipInputStream.getNextEntry()) != null) { + fileSplits.add( + new FtpFileSplit( + 0, zipEntry.getSize(), filePath, zipEntry.getName(), compressType)); + } + return fileSplits; + } catch (Exception e) { + throw new ChunJunRuntimeException(e); + } + } +} diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/spliter/DefaultFileSplit.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/spliter/DefaultFileSplit.java new file mode 100644 index 0000000000..988f3c0d34 --- /dev/null +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/spliter/DefaultFileSplit.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.connector.ftp.spliter; + +import com.dtstack.chunjun.connector.ftp.client.FileUtil; +import com.dtstack.chunjun.connector.ftp.extend.ftp.IFormatConfig; +import com.dtstack.chunjun.connector.ftp.extend.ftp.IFtpHandler; +import com.dtstack.chunjun.connector.ftp.extend.ftp.concurrent.ConcurrentFileSplit; +import com.dtstack.chunjun.connector.ftp.extend.ftp.concurrent.FtpFileSplit; +import com.dtstack.chunjun.throwable.ChunJunRuntimeException; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +public class DefaultFileSplit implements ConcurrentFileSplit { + @Override + public List buildFtpFileSplit( + IFtpHandler handler, IFormatConfig config, List files) { + + List ftpFileSplits = new ArrayList<>(); + + for (String filePath : files) { + try { + String fileName = FileUtil.getFilename(filePath); + long currentFileSize = handler.getFileSize(filePath); + FtpFileSplit fileSplit = new FtpFileSplit(0, currentFileSize, filePath, fileName); + ftpFileSplits.add(fileSplit); + } catch (Exception e) { + throw new ChunJunRuntimeException(e); + } + } + + ftpFileSplits.sort(compare()); + return ftpFileSplits; + } + + /** + * 先根据文件路径排序 再根据文件里面开始的偏移量排序, 从小到大 + * + * @return + */ + @Override + public Comparator compare() { + return Comparator.comparing(FtpFileSplit::getFileAbsolutePath) + .thenComparing( + FtpFileSplit::getStartPosition, + (a, b) -> { + return a.compareTo(b); + }); + } +} diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/table/FtpDynamicTableFactory.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/table/FtpDynamicTableFactory.java index 5f42828d42..32bc4c5a71 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/table/FtpDynamicTableFactory.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/table/FtpDynamicTableFactory.java @@ -47,11 +47,6 @@ import java.util.List; import java.util.Set; -/** - * @program chunjun - * @author: xiuzhu - * @create: 2021/06/19 - */ public class FtpDynamicTableFactory implements DynamicTableSourceFactory, DynamicTableSinkFactory { private static final String IDENTIFIER = "ftp-x"; @@ -65,6 +60,7 @@ private static FtpConfig getFtpConfByOptions(ReadableConfig config) { ftpConfig.setPassword(config.get(FtpOptions.PASSWORD)); ftpConfig.setEncoding(config.get(FtpOptions.ENCODING)); + ftpConfig.setFileType(config.get(FtpOptions.FILE_TYPE)); if (config.get(FtpOptions.TIMEOUT) != null) { ftpConfig.setTimeout(config.get(FtpOptions.TIMEOUT)); @@ -80,6 +76,9 @@ private static FtpConfig getFtpConfByOptions(ReadableConfig config) { ftpConfig.setPort(config.get(FtpOptions.PORT)); } + if (config.get(FtpOptions.isFirstLineHeader) != null) { + ftpConfig.setIsFirstLineHeader(config.get(FtpOptions.isFirstLineHeader)); + } return ftpConfig; } @@ -162,6 +161,8 @@ public Set> optionalOptions() { options.add(FtpOptions.ENCODING); options.add(FtpOptions.MAX_FILE_SIZE); options.add(FtpOptions.FORMAT); + options.add(FtpOptions.isFirstLineHeader); + options.add(FtpOptions.FILE_TYPE); return options; } } diff --git a/chunjun-connectors/chunjun-connector-ftp/src/test/java/com/dtstack/chunjun/connector/ftp/format/TextFileFormatTest.java b/chunjun-connectors/chunjun-connector-ftp/src/test/java/com/dtstack/chunjun/connector/ftp/format/TextFileFormatTest.java deleted file mode 100644 index e3b6e7d47c..0000000000 --- a/chunjun-connectors/chunjun-connector-ftp/src/test/java/com/dtstack/chunjun/connector/ftp/format/TextFileFormatTest.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.chunjun.connector.ftp.format; - -import com.dtstack.chunjun.connector.ftp.client.File; - -import org.apache.commons.io.input.BrokenInputStream; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.io.InputStream; -import java.util.HashMap; - -import static org.junit.Assert.assertThrows; - -public class TextFileFormatTest { - - private TextFileFormat textFileFormatUnderTest; - - @Before - public void setUp() { - textFileFormatUnderTest = new TextFileFormat(); - } - - @Test - public void testOpen_BrokenInputStream() { - // Setup - final File file = - new File("fileCompressPath", "fileAbsolutePath", "fileName", "compressType"); - final InputStream inputStream = new BrokenInputStream(); - final IFormatConfig config = new IFormatConfig(); - config.setFirstLineHeader(false); - config.setFileConfig(new HashMap<>()); - config.setFieldDelimiter("fieldDelimiter"); - config.setEncoding("encoding"); - config.setFields(new String[] {"fields"}); - - // Run the test - assertThrows( - IOException.class, () -> textFileFormatUnderTest.open(file, inputStream, config)); - } - - @Test - public void testClose() throws Exception { - // Setup - // Run the test - textFileFormatUnderTest.close(); - - // Verify the results - } -} diff --git a/chunjun-e2e/src/test/java/com/dtstack/chunjun/connector/test/utils/ChunjunFlinkStandaloneTestEnvironment.java b/chunjun-e2e/src/test/java/com/dtstack/chunjun/connector/test/utils/ChunjunFlinkStandaloneTestEnvironment.java index 548165b01e..0238e4bdeb 100644 --- a/chunjun-e2e/src/test/java/com/dtstack/chunjun/connector/test/utils/ChunjunFlinkStandaloneTestEnvironment.java +++ b/chunjun-e2e/src/test/java/com/dtstack/chunjun/connector/test/utils/ChunjunFlinkStandaloneTestEnvironment.java @@ -34,7 +34,7 @@ import org.apache.flink.table.api.ValidationException; import com.google.common.collect.Lists; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; From d507f59b60be4cef433be92872c677479679dd5c Mon Sep 17 00:00:00 2001 From: superPiKaChu <1623857502@qq.com> Date: Fri, 13 Jan 2023 09:17:14 +0800 Subject: [PATCH 26/41] [BugFix][hbase-connector][#1485] hbase ttl millisecond not second (#1486) (cherry picked from commit 8ff95f2cf8efea0f333c812c4a95d1a51bc7d741) --- .../connector/hbase/converter/HBaseColumnConverter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/converter/HBaseColumnConverter.java b/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/converter/HBaseColumnConverter.java index 132fd94cd9..d020bb8a98 100644 --- a/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/converter/HBaseColumnConverter.java +++ b/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/converter/HBaseColumnConverter.java @@ -213,7 +213,8 @@ public Mutation toExternal(RowData rowData, Mutation output) throws Exception { } put.setTTL( - Optional.ofNullable(hBaseConf.getTtl()).orElseGet(() -> (long) Integer.MAX_VALUE)); + Optional.ofNullable(hBaseConf.getTtl()) + .orElseGet(() -> (long) Integer.MAX_VALUE * 1000)); for (int i = 0; i < rowData.getArity(); i++) { if (rowKeyIndex == i || columnConfigIndex.contains(i)) { From e6eb32ca368f0ba8742673ec9696c41e633764b9 Mon Sep 17 00:00:00 2001 From: Paddy Date: Sat, 28 Jan 2023 19:14:31 +0800 Subject: [PATCH 27/41] [feat][jdbc]improvement jdbc read/write performance (#1488) (cherry picked from commit afc7d519ebed5bd90fb67a7e5ba77bba1ab44fe2) --- .../greenplum/dialect/GreenplumDialect.java | 3 +- .../greenplum/sink/GreenplumOutputFormat.java | 246 ++++++++++++++++++ .../greenplum/sink/GreenplumSinkFactory.java | 34 ++- .../chunjun/connector/jdbc/conf/JdbcConf.java | 22 ++ .../connector/jdbc/sink/JdbcOutputFormat.java | 15 +- .../jdbc/sink/JdbcOutputFormatBuilder.java | 8 + .../jdbc/source/JdbcDynamicTableSource.java | 2 +- .../jdbc/source/JdbcInputFormat.java | 6 + .../converter/OracleColumnConverter.java | 204 +++++++++++++-- .../oracle/source/OracleInputFormat.java | 75 +++++- .../postgresql/dialect/PostgresqlDialect.java | 23 +- .../postgresql/sink/PostgresOutputFormat.java | 1 + .../vertica11/sink/Vertica11OutputFormat.java | 5 +- 13 files changed, 595 insertions(+), 49 deletions(-) create mode 100644 chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/sink/GreenplumOutputFormat.java diff --git a/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/dialect/GreenplumDialect.java b/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/dialect/GreenplumDialect.java index 942af88c1a..62045c9e0b 100644 --- a/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/dialect/GreenplumDialect.java +++ b/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/dialect/GreenplumDialect.java @@ -33,7 +33,8 @@ public class GreenplumDialect extends PostgresqlDialect { private static final String DIALECT_NAME = "Greenplum"; private static final String DRIVER = "com.pivotal.jdbc.GreenplumDriver"; - private static final String URL_START = "jdbc:pivotal:greenplum:"; + public static final String URL_START = "jdbc:pivotal:greenplum:"; + public static final String DATABASE_NAME = ";DatabaseName="; @Override public String dialectName() { diff --git a/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/sink/GreenplumOutputFormat.java b/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/sink/GreenplumOutputFormat.java new file mode 100644 index 0000000000..9c19d5cdfc --- /dev/null +++ b/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/sink/GreenplumOutputFormat.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.connector.greenplum.sink; + +import com.dtstack.chunjun.connector.jdbc.converter.JdbcColumnConverter; +import com.dtstack.chunjun.connector.jdbc.sink.JdbcOutputFormat; +import com.dtstack.chunjun.connector.postgresql.converter.PostgresqlColumnConverter; +import com.dtstack.chunjun.connector.postgresql.dialect.PostgresqlDialect; +import com.dtstack.chunjun.constants.ConstantValue; +import com.dtstack.chunjun.element.ColumnRowData; +import com.dtstack.chunjun.enums.EWriteMode; +import com.dtstack.chunjun.throwable.NoRestartException; +import com.dtstack.chunjun.throwable.WriteRecordException; + +import org.apache.flink.streaming.api.CheckpointingMode; +import org.apache.flink.table.data.RowData; + +import org.apache.commons.lang3.math.NumberUtils; +import org.postgresql.copy.CopyManager; +import org.postgresql.core.BaseConnection; + +import java.io.ByteArrayInputStream; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.SQLException; + +/** + * @program: flinkx + * @author: jier + */ +public class GreenplumOutputFormat extends JdbcOutputFormat { + + // pg 字符串里含有\u0000 会报错 ERROR: invalid byte sequence for encoding "UTF8": 0x00 + public static final String SPACE = "\u0000"; + + private static final String LINE_DELIMITER = "\n"; + private CopyManager copyManager; + private boolean disableCopyMode = false; + private String copySql = ""; + public static final String INSERT_SQL_MODE_TYPE = "copy"; + private static final String DEFAULT_FIELD_DELIMITER = "\001"; + + private static final String DEFAULT_NULL_VALUE = "\002"; + + /** 数据源类型信息 * */ + private final String dbType = DbType.POSTGRESQL.name(); + + @Override + protected void openInternal(int taskNumber, int numTasks) { + super.openInternal(taskNumber, numTasks); + try { + // check is use copy mode for insert + disableCopyMode = + jdbcConf.getInsertSqlMode() != null + && !INSERT_SQL_MODE_TYPE.equalsIgnoreCase(jdbcConf.getInsertSqlMode()); + if (EWriteMode.INSERT.name().equalsIgnoreCase(jdbcConf.getMode()) && !disableCopyMode) { + LOG.info("will use copy mode"); + copyManager = new CopyManager((BaseConnection) dbConn); + + PostgresqlDialect pgDialect = (PostgresqlDialect) jdbcDialect; + copySql = + pgDialect.getCopyStatement( + jdbcConf.getSchema(), + jdbcConf.getTable(), + columnNameList.toArray(new String[0]), + DEFAULT_FIELD_DELIMITER, + DEFAULT_NULL_VALUE); + + LOG.info("write sql:{}", copySql); + } + checkUpsert(); + if (rowConverter instanceof PostgresqlColumnConverter + && dbConn instanceof BaseConnection) { + ((PostgresqlColumnConverter) rowConverter).setConnection((BaseConnection) dbConn); + } + } catch (SQLException sqe) { + throw new IllegalArgumentException("checkUpsert() failed.", sqe); + } + } + + @Override + protected void writeSingleRecordInternal(RowData row) throws WriteRecordException { + if (disableCopyMode) { + super.writeSingleRecordInternal(row); + } else { + if (rowConverter instanceof JdbcColumnConverter) { + ColumnRowData colRowData = (ColumnRowData) row; + // write with copy + int index = 0; + try { + StringBuilder rowStr = new StringBuilder(); + int lastIndex = row.getArity() - 1; + for (; index < row.getArity(); index++) { + appendColumn(colRowData, index, rowStr, index == lastIndex); + } + String rowVal = copyModeReplace(rowStr.toString()); + try (ByteArrayInputStream bi = + new ByteArrayInputStream(rowVal.getBytes(StandardCharsets.UTF_8))) { + copyManager.copyIn(copySql, bi); + } + } catch (Exception e) { + processWriteException(e, index, row); + } + } else { + throw new NoRestartException("copy mode only support data sync with out table"); + } + } + } + + @Override + protected void writeMultipleRecordsInternal() throws Exception { + if (disableCopyMode) { + super.writeMultipleRecordsInternal(); + } else { + if (rowConverter instanceof JdbcColumnConverter) { + StringBuilder rowsStrBuilder = new StringBuilder(128); + for (RowData row : rows) { + ColumnRowData colRowData = (ColumnRowData) row; + int lastIndex = row.getArity() - 1; + StringBuilder rowStr = new StringBuilder(128); + for (int index = 0; index < row.getArity(); index++) { + appendColumn(colRowData, index, rowStr, index == lastIndex); + } + String tempData = rowStr.toString(); + rowsStrBuilder.append(copyModeReplace(tempData)).append(LINE_DELIMITER); + } + String rowVal = rowsStrBuilder.toString(); + try (ByteArrayInputStream bi = + new ByteArrayInputStream(rowVal.getBytes(StandardCharsets.UTF_8))) { + copyManager.copyIn(copySql, bi); + if (checkpointEnabled && CheckpointingMode.EXACTLY_ONCE == checkpointMode) { + rowsOfCurrentTransaction += rows.size(); + } + } + } else { + throw new NoRestartException("copy mode only support data sync with out table"); + } + } + } + + private void appendColumn( + ColumnRowData colRowData, int pos, StringBuilder rowStr, boolean isLast) { + Object col = colRowData.getField(pos); + if (col == null) { + rowStr.append(DEFAULT_NULL_VALUE); + } else { + rowStr.append(col); + } + if (!isLast) { + rowStr.append(DEFAULT_FIELD_DELIMITER); + } + } + + /** + * \r \n \ 等特殊字符串需要转义 + * + * @return + */ + private String copyModeReplace(String rowStr) { + if (rowStr.contains("\\")) { + rowStr = rowStr.replaceAll("\\\\", "\\\\\\\\"); + } + if (rowStr.contains("\r")) { + rowStr = rowStr.replaceAll("\r", "\\\\r"); + } + + if (rowStr.contains("\n")) { + rowStr = rowStr.replaceAll("\n", "\\\\n"); + } + + // pg 字符串里含有\u0000 会报错 ERROR: invalid byte sequence for encoding "UTF8": 0x00 + if (rowStr.contains(SPACE)) { + rowStr = rowStr.replaceAll(SPACE, ""); + } + return rowStr; + } + + /** 数据源类型 * */ + public enum DbType { + POSTGRESQL, + ADB + } + + /** + * 当mode为update时进行校验 + * + * @return + * @throws SQLException + */ + public void checkUpsert() throws SQLException { + if (EWriteMode.UPDATE.name().equalsIgnoreCase(jdbcConf.getMode())) { + try (Connection connection = getConnection()) { + + // 效验版本 + String databaseProductVersion = + connection.getMetaData().getDatabaseProductVersion(); + LOG.info("source version is {}", databaseProductVersion); + String[] split = databaseProductVersion.split("\\."); + // 10.1.12 + if (split.length > 2) { + databaseProductVersion = split[0] + ConstantValue.POINT_SYMBOL + split[1]; + } + + if (NumberUtils.isNumber(databaseProductVersion)) { + BigDecimal sourceVersion = new BigDecimal(databaseProductVersion); + if (dbType.equalsIgnoreCase(DbType.POSTGRESQL.name())) { + // pg大于等于9.5 + if (sourceVersion.compareTo(new BigDecimal("9.5")) < 0) { + throw new RuntimeException( + "the postgreSql version is [" + + databaseProductVersion + + "] and must greater than or equal to 9.5 when you use update mode and source is " + + DbType.POSTGRESQL.name()); + } + } else if (dbType.equalsIgnoreCase(DbType.ADB.name())) { + // adb大于等于9.4 + if (sourceVersion.compareTo(new BigDecimal("9.4")) < 0) { + throw new RuntimeException( + "the postgreSql version is [" + + databaseProductVersion + + "] and must greater than or equal to 9.4 when you use update mode and source is " + + DbType.ADB.name()); + } + } + } + } + } + } +} diff --git a/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/sink/GreenplumSinkFactory.java b/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/sink/GreenplumSinkFactory.java index d0aa9d0ae1..fa38a5d998 100644 --- a/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/sink/GreenplumSinkFactory.java +++ b/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/sink/GreenplumSinkFactory.java @@ -20,16 +20,44 @@ import com.dtstack.chunjun.conf.SyncConf; import com.dtstack.chunjun.connector.greenplum.dialect.GreenplumDialect; -import com.dtstack.chunjun.connector.postgresql.sink.PostgresqlSinkFactory; +import com.dtstack.chunjun.connector.jdbc.sink.JdbcOutputFormatBuilder; +import com.dtstack.chunjun.connector.jdbc.sink.JdbcSinkFactory; +import com.dtstack.chunjun.connector.postgresql.dialect.PostgresqlDialect; + +import org.apache.commons.lang.StringUtils; + +import static com.dtstack.chunjun.connector.greenplum.sink.GreenplumOutputFormat.INSERT_SQL_MODE_TYPE; /** * company www.dtstack.com * * @author jier */ -public class GreenplumSinkFactory extends PostgresqlSinkFactory { +public class GreenplumSinkFactory extends JdbcSinkFactory { public GreenplumSinkFactory(SyncConf syncConf) { - super(syncConf, new GreenplumDialect()); + super(syncConf, null); + if (syncConf.getWriter().getParameter().get("insertSqlMode") != null + && INSERT_SQL_MODE_TYPE.equalsIgnoreCase( + syncConf.getWriter().getParameter().get("insertSqlMode").toString())) { + this.jdbcDialect = new PostgresqlDialect(); + String pgUrl = changeToPostgresqlUrl(this.jdbcConf.getJdbcUrl()); + this.jdbcConf.setJdbcUrl(pgUrl); + } else { + this.jdbcDialect = new GreenplumDialect(); + } + } + + @Override + protected JdbcOutputFormatBuilder getBuilder() { + return new JdbcOutputFormatBuilder(new GreenplumOutputFormat()); + } + + private String changeToPostgresqlUrl(String gpUrl) { + String pgUrl = + StringUtils.replaceOnce( + gpUrl, GreenplumDialect.URL_START, PostgresqlDialect.URL_START); + pgUrl = StringUtils.replaceOnce(pgUrl, GreenplumDialect.DATABASE_NAME, "/"); + return pgUrl; } } diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/conf/JdbcConf.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/conf/JdbcConf.java index dfe13a856d..a3f4ca1b38 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/conf/JdbcConf.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/conf/JdbcConf.java @@ -106,6 +106,10 @@ public class JdbcConf extends ChunJunCommonConf implements Serializable { /** upsert 写数据库时，是否null覆盖原来的值 */ protected boolean allReplace = false; + protected boolean isAutoCommit = false; + + private boolean defineColumnTypeForStatement = false; + public Boolean getInitReporter() { return initReporter; } @@ -419,6 +423,14 @@ public void setAllReplace(boolean allReplace) { this.allReplace = allReplace; } + public boolean isAutoCommit() { + return isAutoCommit; + } + + public boolean isDefineColumnTypeForStatement() { + return defineColumnTypeForStatement; + } + public String getSplitStrategy() { return splitStrategy; } @@ -485,9 +497,13 @@ public String toString() { + increment + ", polling=" + polling + + ", pollingFromMax=" + + pollingFromMax + ", increColumn='" + increColumn + '\'' + + ", isOrderBy=" + + isOrderBy + ", increColumnIndex=" + increColumnIndex + ", increColumnType='" @@ -508,6 +524,8 @@ public String toString() { + restoreColumnIndex + ", useMaxFunc=" + useMaxFunc + + ", initReporter=" + + initReporter + ", mode='" + mode + '\'' @@ -521,6 +539,10 @@ public String toString() { + updateKey + ", allReplace=" + allReplace + + ", isAutoCommit=" + + isAutoCommit + + ", defineColumnTypeForStatement=" + + defineColumnTypeForStatement + '}'; } } diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormat.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormat.java index e2db65adf5..2dd7433c38 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormat.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormat.java @@ -66,7 +66,6 @@ public class JdbcOutputFormat extends BaseRichOutputFormat { protected JdbcDialect jdbcDialect; protected transient Connection dbConn; - protected boolean autoCommit = true; protected transient PreparedStmtProxy stmtProxy; @@ -87,10 +86,7 @@ protected void openInternal(int taskNumber, int numTasks) { try { dbConn = getConnection(); // 默认关闭事务自动提交，手动控制事务 - if (Semantic.EXACTLY_ONCE == semantic) { - autoCommit = false; - dbConn.setAutoCommit(autoCommit); - } + dbConn.setAutoCommit(jdbcConf.isAutoCommit()); if (!EWriteMode.INSERT.name().equalsIgnoreCase(jdbcConf.getMode())) { List updateKey = jdbcConf.getUniqueKey(); if (CollectionUtils.isEmpty(updateKey)) { @@ -135,6 +131,11 @@ protected void writeSingleRecordInternal(RowData row) throws WriteRecordExceptio int index = 0; try { stmtProxy.writeSingleRecordInternal(row); + if (Semantic.EXACTLY_ONCE == semantic) { + rowsOfCurrentTransaction += rows.size(); + } else { + JdbcUtil.commit(dbConn); + } } catch (Exception e) { JdbcUtil.rollBack(dbConn); processWriteException(e, index, row); @@ -164,6 +165,8 @@ protected void writeMultipleRecordsInternal() throws Exception { // 开启了cp，但是并没有使用2pc方式让下游数据可见 if (Semantic.EXACTLY_ONCE == semantic) { rowsOfCurrentTransaction += rows.size(); + } else { + JdbcUtil.commit(dbConn); } } catch (Exception e) { LOG.warn( @@ -216,7 +219,7 @@ public void rollback(long checkpointId) throws Exception { public void doCommit() throws SQLException { try { - if (!autoCommit) { + if (!jdbcConf.isAutoCommit()) { dbConn.commit(); } snapshotWriteCounter.add(rowsOfCurrentTransaction); diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormatBuilder.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormatBuilder.java index 355a41162f..7b75737cbd 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormatBuilder.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormatBuilder.java @@ -20,6 +20,7 @@ import com.dtstack.chunjun.connector.jdbc.conf.JdbcConf; import com.dtstack.chunjun.connector.jdbc.dialect.JdbcDialect; import com.dtstack.chunjun.converter.AbstractRowConverter; +import com.dtstack.chunjun.enums.Semantic; import com.dtstack.chunjun.sink.format.BaseRichOutputFormatBuilder; import org.apache.commons.lang3.StringUtils; @@ -66,6 +67,13 @@ protected void checkFormat() { if (StringUtils.isBlank(jdbcConf.getJdbcUrl())) { sb.append("No jdbc url supplied;\n"); } + + if (Semantic.getByName(jdbcConf.getSemantic()) == Semantic.EXACTLY_ONCE + && jdbcConf.isAutoCommit()) { + sb.append( + "Exactly-once semantics requires that the jdbc driver is not in auto-commit mode;\n"); + } + if (sb.length() > 0) { throw new IllegalArgumentException(sb.toString()); } diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/source/JdbcDynamicTableSource.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/source/JdbcDynamicTableSource.java index 93893a8cbf..b33ec6832f 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/source/JdbcDynamicTableSource.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/source/JdbcDynamicTableSource.java @@ -154,7 +154,7 @@ public ScanRuntimeProvider getScanRuntimeProvider(ScanContext runtimeProviderCon if (StringUtils.isNotBlank(splitPk)) { FieldConf fieldConf = FieldConf.getSameNameMetaColumn(jdbcConf.getColumn(), splitPk); if (fieldConf != null) { - jdbcConf.setSplitPk(fieldConf.getType()); + jdbcConf.setSplitPk(fieldConf.getName()); splitKeyUtil = jdbcDialect.initKeyUtil(fieldConf.getName(), fieldConf.getType()); } } diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/source/JdbcInputFormat.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/source/JdbcInputFormat.java index 63f48f3ea6..06a30b27f9 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/source/JdbcInputFormat.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/source/JdbcInputFormat.java @@ -707,6 +707,10 @@ protected void executeQuery(String startLocation) throws SQLException { } } else { statement = dbConn.createStatement(resultSetType, resultSetConcurrency); + if (jdbcConf.isDefineColumnTypeForStatement() + && StringUtils.isBlank(jdbcConf.getCustomSql())) { + defineColumnType(statement); + } statement.setFetchSize(jdbcConf.getFetchSize()); statement.setQueryTimeout(jdbcConf.getQueryTimeOut()); resultSet = statement.executeQuery(jdbcConf.getQuerySql()); @@ -714,6 +718,8 @@ protected void executeQuery(String startLocation) throws SQLException { } } + protected void defineColumnType(Statement statement) throws SQLException {} + /** init prepareStatement */ public void initPrepareStatement(String querySql) throws SQLException { ps = dbConn.prepareStatement(querySql, resultSetType, resultSetConcurrency); diff --git a/chunjun-connectors/chunjun-connector-oracle/src/main/java/com/dtstack/chunjun/connector/oracle/converter/OracleColumnConverter.java b/chunjun-connectors/chunjun-connector-oracle/src/main/java/com/dtstack/chunjun/connector/oracle/converter/OracleColumnConverter.java index c9287ae1c8..e51ac41f7d 100644 --- a/chunjun-connectors/chunjun-connector-oracle/src/main/java/com/dtstack/chunjun/connector/oracle/converter/OracleColumnConverter.java +++ b/chunjun-connectors/chunjun-connector-oracle/src/main/java/com/dtstack/chunjun/connector/oracle/converter/OracleColumnConverter.java @@ -19,10 +19,13 @@ package com.dtstack.chunjun.connector.oracle.converter; import com.dtstack.chunjun.conf.ChunJunCommonConf; +import com.dtstack.chunjun.conf.FieldConf; import com.dtstack.chunjun.connector.jdbc.converter.JdbcColumnConverter; import com.dtstack.chunjun.connector.jdbc.statement.FieldNamedPreparedStatement; +import com.dtstack.chunjun.constants.ConstantValue; import com.dtstack.chunjun.converter.IDeserializationConverter; import com.dtstack.chunjun.converter.ISerializationConverter; +import com.dtstack.chunjun.element.AbstractBaseColumn; import com.dtstack.chunjun.element.ColumnRowData; import com.dtstack.chunjun.element.column.BigDecimalColumn; import com.dtstack.chunjun.element.column.BooleanColumn; @@ -31,17 +34,25 @@ import com.dtstack.chunjun.element.column.TimestampColumn; import org.apache.flink.connector.jdbc.utils.JdbcTypeUtil; +import org.apache.flink.table.data.RowData; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.TimestampType; import org.apache.flink.table.types.utils.TypeConversions; import oracle.sql.TIMESTAMP; +import org.apache.commons.lang3.StringUtils; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.io.StringReader; -import java.math.BigDecimal; +import java.sql.ResultSet; import java.sql.Timestamp; +import java.time.ZoneId; +import java.util.Calendar; +import java.util.List; +import java.util.Locale; +import java.util.TimeZone; /** * company www.dtstack.com @@ -50,53 +61,167 @@ */ public class OracleColumnConverter extends JdbcColumnConverter { + private static final Calendar var4 = + Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.US); + private int currentIndex = 0; + public OracleColumnConverter(RowType rowType, ChunJunCommonConf commonConf) { super(rowType, commonConf); } + @Override + @SuppressWarnings("unchecked") + public RowData toInternal(ResultSet resultSet) throws Exception { + List fieldConfList = commonConf.getColumn(); + ColumnRowData result; + if (fieldConfList.size() == 1 + && ConstantValue.STAR_SYMBOL.equals(fieldConfList.get(0).getName())) { + result = new ColumnRowData(fieldTypes.length); + for (int index = 0; index < fieldTypes.length; index++) { + Object field = resultSet.getObject(index + 1); + AbstractBaseColumn baseColumn = + (AbstractBaseColumn) toInternalConverters.get(index).deserialize(field); + result.addField(baseColumn); + } + return result; + } + currentIndex = 0; + result = new ColumnRowData(fieldConfList.size()); + for (FieldConf fieldConf : fieldConfList) { + AbstractBaseColumn baseColumn = null; + if (StringUtils.isBlank(fieldConf.getValue())) { + baseColumn = + (AbstractBaseColumn) + toInternalConverters.get(currentIndex).deserialize(resultSet); + currentIndex++; + } + result.addField(assembleFieldProps(fieldConf, baseColumn)); + } + return result; + } + @Override protected IDeserializationConverter createInternalConverter(LogicalType type) { switch (type.getTypeRoot()) { case BOOLEAN: - return val -> new BooleanColumn(Boolean.parseBoolean(val.toString())); + return (IDeserializationConverter) + field -> { + Object object = field.getObject(currentIndex + 1); + if (object == null) { + return null; + } + return new BooleanColumn(Boolean.parseBoolean(object.toString())); + }; case TINYINT: - return val -> new BigDecimalColumn(((Integer) val).byteValue()); + return (IDeserializationConverter) + field -> { + Object object = field.getObject(currentIndex + 1); + if (object == null) { + return null; + } + return new BigDecimalColumn(((Integer) object).byteValue()); + }; case SMALLINT: case INTEGER: - return val -> new BigDecimalColumn((Integer) val); case FLOAT: - return val -> new BigDecimalColumn((Float) val); case DOUBLE: - return val -> new BigDecimalColumn((Double) val); case BIGINT: - return val -> new BigDecimalColumn((Long) val); case DECIMAL: - return val -> new BigDecimalColumn((BigDecimal) val); + return (IDeserializationConverter) + field -> { + String object = field.getString(currentIndex + 1); + if (object == null) { + return null; + } + return new BigDecimalColumn(object); + }; + case CHAR: case VARCHAR: if (type instanceof ClobType) { - return val -> { - oracle.sql.CLOB clob = (oracle.sql.CLOB) val; - return new StringColumn(ConvertUtil.convertClob(clob)); - }; + return (IDeserializationConverter) + field -> { + Object object = field.getObject(currentIndex + 1); + if (object == null) { + return null; + } + oracle.sql.CLOB clob = (oracle.sql.CLOB) object; + return new StringColumn(ConvertUtil.convertClob(clob)); + }; } - return val -> new StringColumn((String) val); + return (IDeserializationConverter) + field -> { + String object = field.getString(currentIndex + 1); + if (object == null) { + return null; + } + return new StringColumn(object); + }; case DATE: - return val -> new TimestampColumn((Timestamp) val, 0); - case TIMESTAMP_WITH_TIME_ZONE: + return (IDeserializationConverter) + field -> { + Timestamp object = field.getTimestamp(currentIndex + 1); + if (object == null) { + return null; + } + return new TimestampColumn(object, 0); + }; case TIMESTAMP_WITHOUT_TIME_ZONE: - return val -> new TimestampColumn(((TIMESTAMP) val).timestampValue()); + return (IDeserializationConverter) + field -> { + int precision = ((TimestampType) (type)).getPrecision(); + Timestamp object = field.getTimestamp(currentIndex + 1); + if (object == null) { + return null; + } + return new TimestampColumn(object, precision); + }; + case TIMESTAMP_WITH_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return (IDeserializationConverter) + field -> { + Object val = field.getObject(currentIndex + 1); + if (val == null) { + return null; + } + if (val instanceof TIMESTAMP) { + return new TimestampColumn(((TIMESTAMP) val).timestampValue()); + } else if (val instanceof oracle.sql.TIMESTAMPLTZ) { + return new TimestampColumn( + ((oracle.sql.TIMESTAMPLTZ) val) + .timestampValue(null, Calendar.getInstance())); + } else if (val instanceof oracle.sql.TIMESTAMPTZ) { + return new TimestampColumn( + (toTimestamp2( + ((oracle.sql.TIMESTAMPTZ) val).getBytes(), null))); + } else if (val instanceof Timestamp) { + return new TimestampColumn((Timestamp) val); + } + throw new RuntimeException("not support type" + val.getClass()); + }; case BINARY: case VARBINARY: - return val -> { - if (type instanceof BlobType) { - oracle.sql.BLOB blob = (oracle.sql.BLOB) val; - byte[] bytes = ConvertUtil.toByteArray(blob); - return new BytesColumn(bytes); - } else { - return new BytesColumn((byte[]) val); - } - }; + if (type instanceof BlobType) { + return (IDeserializationConverter) + field -> { + Object object = field.getObject(currentIndex + 1); + if (object == null) { + return null; + } + oracle.sql.BLOB blob = (oracle.sql.BLOB) object; + byte[] bytes = ConvertUtil.toByteArray(blob); + return new BytesColumn(bytes); + }; + } else { + return (IDeserializationConverter) + field -> { + byte[] object = field.getBytes(currentIndex + 1); + if (object == null) { + return null; + } + return new BytesColumn(object); + }; + } default: throw new UnsupportedOperationException("Unsupported type:" + type); } @@ -169,6 +294,7 @@ protected ISerializationConverter createExternalCon case DATE: case TIMESTAMP_WITH_TIME_ZONE: case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: return (val, index, statement) -> statement.setTimestamp( index, ((ColumnRowData) val).getField(index).asTimestamp()); @@ -188,4 +314,32 @@ protected ISerializationConverter createExternalCon throw new UnsupportedOperationException("Unsupported type:" + type); } } + + public static Timestamp toTimestamp2(byte[] var1, String zone) { + int[] var2 = new int[13]; + + int var3; + for (var3 = 0; var3 < 13; ++var3) { + var2[var3] = var1[var3] & 255; + } + + var3 = TIMESTAMP.getJavaYear(var2[0], var2[1]); + + var4.clear(); + var4.set(1, var3); + var4.set(2, var2[2] - 1); + var4.set(5, var2[3]); + var4.set(11, var2[4] - 1); + var4.set(12, var2[5] - 1); + var4.set(13, var2[6] - 1); + var4.set(14, 0); + long var5 = var4.getTime().getTime(); + Timestamp var7 = new Timestamp(var5); + int var8 = TIMESTAMP.getNanos(var1, 7); + var7.setNanos(var8); + if (StringUtils.isNotBlank(zone)) { + var7 = Timestamp.valueOf(var7.toInstant().atZone(ZoneId.of(zone)).toLocalDateTime()); + } + return var7; + } } diff --git a/chunjun-connectors/chunjun-connector-oracle/src/main/java/com/dtstack/chunjun/connector/oracle/source/OracleInputFormat.java b/chunjun-connectors/chunjun-connector-oracle/src/main/java/com/dtstack/chunjun/connector/oracle/source/OracleInputFormat.java index df61ce1067..a33990b5a8 100644 --- a/chunjun-connectors/chunjun-connector-oracle/src/main/java/com/dtstack/chunjun/connector/oracle/source/OracleInputFormat.java +++ b/chunjun-connectors/chunjun-connector-oracle/src/main/java/com/dtstack/chunjun/connector/oracle/source/OracleInputFormat.java @@ -19,10 +19,83 @@ package com.dtstack.chunjun.connector.oracle.source; import com.dtstack.chunjun.connector.jdbc.source.JdbcInputFormat; +import com.dtstack.chunjun.connector.oracle.converter.BlobType; +import com.dtstack.chunjun.connector.oracle.converter.ClobType; +import com.dtstack.chunjun.util.TableUtil; + +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; + +import oracle.jdbc.OracleStatement; +import oracle.jdbc.OracleTypes; + +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.util.ArrayList; /** * company www.dtstack.com * * @author jier */ -public class OracleInputFormat extends JdbcInputFormat {} +public class OracleInputFormat extends JdbcInputFormat { + + @Override + protected void defineColumnType(Statement statement) throws SQLException { + if (statement instanceof oracle.jdbc.OracleStatement) { + OracleStatement oracleStatement = (OracleStatement) statement; + ArrayList names = new ArrayList<>(); + ArrayList types = new ArrayList<>(); + jdbcConf.getColumn() + .forEach( + i -> { + names.add(i.getName()); + types.add(i.getType()); + }); + RowType rowType = + TableUtil.createRowType(names, types, jdbcDialect.getRawTypeConverter()); + OracleStatement oraclePreparedStatement = (OracleStatement) statement; + oraclePreparedStatement.clearDefines(); + for (int i = 0; i < rowType.getFieldCount(); i++) { + defineType(i, rowType.getTypeAt(i), oracleStatement); + } + } + } + + protected void defineType(int position, LogicalType logicalType, OracleStatement statement) + throws SQLException { + switch (logicalType.getTypeRoot()) { + case SMALLINT: + case INTEGER: + case FLOAT: + case DOUBLE: + case BIGINT: + case DECIMAL: + statement.defineColumnType(position + 1, Types.VARCHAR); + break; + case CHAR: + case VARCHAR: + if (!(logicalType instanceof ClobType)) { + statement.defineColumnType(position + 1, Types.VARCHAR); + } + break; + case DATE: + case TIMESTAMP_WITHOUT_TIME_ZONE: + statement.defineColumnType(position + 1, Types.TIMESTAMP); + break; + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + statement.defineColumnType(position + 1, OracleTypes.TIMESTAMPLTZ); + break; + case TIMESTAMP_WITH_TIME_ZONE: + statement.defineColumnType(position + 1, OracleTypes.TIMESTAMPTZ); + break; + case BINARY: + case VARBINARY: + if (!(logicalType instanceof BlobType)) { + statement.defineColumnType(position + 1, Types.VARBINARY); + } + break; + } + } +} diff --git a/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/dialect/PostgresqlDialect.java b/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/dialect/PostgresqlDialect.java index 783e5fe56f..87c3f04125 100644 --- a/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/dialect/PostgresqlDialect.java +++ b/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/dialect/PostgresqlDialect.java @@ -47,9 +47,9 @@ public class PostgresqlDialect implements JdbcDialect { private static final String DIALECT_NAME = "PostgreSQL"; private static final String DRIVER = "org.postgresql.Driver"; - private static final String URL_START = "jdbc:postgresql:"; + public static final String URL_START = "jdbc:postgresql:"; - private static final String COPY_SQL_TEMPL = + protected static final String COPY_SQL_TEMPL = "copy %s(%s) from stdin DELIMITER '%s' NULL as '%s'"; @Override @@ -140,15 +140,22 @@ public String getSelectFromStatement( } public String getCopyStatement( - String tableName, String[] fields, String fieldDelimiter, String nullVal) { + String schemaName, + String tableName, + String[] fields, + String fieldDelimiter, + String nullVal) { String fieldsExpression = Arrays.stream(fields).map(this::quoteIdentifier).collect(Collectors.joining(", ")); + String tableLocation; + if (schemaName != null && !"".equals(schemaName.trim())) { + tableLocation = quoteIdentifier(schemaName) + "." + quoteIdentifier(tableName); + } else { + tableLocation = quoteIdentifier(tableName); + } + return String.format( - COPY_SQL_TEMPL, - quoteIdentifier(tableName), - fieldsExpression, - fieldDelimiter, - nullVal); + COPY_SQL_TEMPL, tableLocation, fieldsExpression, fieldDelimiter, nullVal); } } diff --git a/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/sink/PostgresOutputFormat.java b/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/sink/PostgresOutputFormat.java index f39d02b38e..54161cdb98 100644 --- a/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/sink/PostgresOutputFormat.java +++ b/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/sink/PostgresOutputFormat.java @@ -76,6 +76,7 @@ protected void openInternal(int taskNumber, int numTasks) { PostgresqlDialect pgDialect = (PostgresqlDialect) jdbcDialect; copySql = pgDialect.getCopyStatement( + jdbcConf.getSchema(), jdbcConf.getTable(), columnNameList.toArray(new String[0]), StringUtils.isNullOrWhitespaceOnly(jdbcConf.getFieldDelim().trim()) diff --git a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/sink/Vertica11OutputFormat.java b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/sink/Vertica11OutputFormat.java index 2d0b90a2ba..004164aafe 100644 --- a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/sink/Vertica11OutputFormat.java +++ b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/sink/Vertica11OutputFormat.java @@ -23,7 +23,6 @@ import com.dtstack.chunjun.connector.jdbc.util.JdbcUtil; import com.dtstack.chunjun.connector.vertica11.dialect.Vertica11Dialect; import com.dtstack.chunjun.enums.EWriteMode; -import com.dtstack.chunjun.enums.Semantic; import com.dtstack.chunjun.util.JsonUtil; import org.apache.commons.collections.CollectionUtils; @@ -39,9 +38,7 @@ protected void openInternal(int taskNumber, int numTasks) { dbConn = getConnection(); // By default, transaction auto-commit is turned off, and transactions are manually // controlled - if (Semantic.EXACTLY_ONCE == semantic) { - dbConn.setAutoCommit(false); - } + dbConn.setAutoCommit(jdbcConf.isAutoCommit()); if (!EWriteMode.INSERT.name().equalsIgnoreCase(jdbcConf.getMode())) { List updateKey = jdbcConf.getUniqueKey(); if (CollectionUtils.isEmpty(updateKey)) { From 6bc3a2e0d8228fc761b3edf3e0dfedacd6212539 Mon Sep 17 00:00:00 2001 From: wangJm Date: Sun, 29 Jan 2023 15:20:55 +0800 Subject: [PATCH 28/41] [feat-#1487][iceberg]add iceberg writer (#1489) (cherry picked from commit de5b214e3f0129bbc520d9121f55b14360bfeea2) --- .../chunjun-connector-iceberg/pom.xml | 67 ++++--- .../connector/iceberg/conf/IcebergConf.java | 69 +++++++ .../iceberg/conf/IcebergReaderConf.java | 69 ------- .../iceberg/conf/IcebergWriterConf.java | 96 ---------- .../converter/IcebergColumnConverter.java | 179 ------------------ .../converter/IcebergRawTypeConverter.java | 63 ------ .../sink/ChunjunRowDataConvertMap.java | 98 ++++++++++ .../sink/IcebergMetricsMapFunction.java | 167 ---------------- .../iceberg/sink/IcebergSinkFactory.java | 141 ++++++-------- .../iceberg/source/IcebergInputFormat.java | 84 -------- .../source/IcebergInputFormatBuilder.java | 48 ----- .../iceberg/source/IcebergSourceFactory.java | 87 --------- .../json/iceberg/iceberg_stream.json | 72 ------- .../json/iceberg/stream_iceberg.json | 36 ++-- chunjun-local-test/pom.xml | 6 + .../iceberg/iceberg-sink.md" | 45 +++++ 16 files changed, 334 insertions(+), 993 deletions(-) create mode 100644 chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/conf/IcebergConf.java delete mode 100644 chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/conf/IcebergReaderConf.java delete mode 100644 chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/conf/IcebergWriterConf.java delete mode 100644 chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/converter/IcebergColumnConverter.java delete mode 100644 chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/converter/IcebergRawTypeConverter.java create mode 100644 chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/sink/ChunjunRowDataConvertMap.java delete mode 100644 chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/sink/IcebergMetricsMapFunction.java delete mode 100644 chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/source/IcebergInputFormat.java delete mode 100644 chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/source/IcebergInputFormatBuilder.java delete mode 100644 chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/source/IcebergSourceFactory.java delete mode 100644 chunjun-examples/json/iceberg/iceberg_stream.json create mode 100644 "docs_zh/ChunJun\350\277\236\346\216\245\345\231\250/iceberg/iceberg-sink.md" diff --git a/chunjun-connectors/chunjun-connector-iceberg/pom.xml b/chunjun-connectors/chunjun-connector-iceberg/pom.xml index a2bd6752a3..722ba9cd3b 100644 --- a/chunjun-connectors/chunjun-connector-iceberg/pom.xml +++ b/chunjun-connectors/chunjun-connector-iceberg/pom.xml @@ -31,40 +31,53 @@ chunjun-connector-iceberg ChunJun : Connectors : Iceberg - + + 2.7.5 + 0.13.1 + + - - - - - - - - - - - - + org.apache.iceberg - iceberg-flink-runtime-1.12 - 0.13.2 + iceberg-flink-runtime-1.13 + ${iceberg.version} + + + org.apache.hadoop + hadoop-hdfs + + + + org.apache.hadoop + hadoop-common + + + + org.apache.hadoop + hadoop-client + + + + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} - - - - - - - - - - - + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + org.apache.hadoop + hadoop-client + ${hadoop.version} + - diff --git a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/conf/IcebergConf.java b/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/conf/IcebergConf.java new file mode 100644 index 0000000000..046274f1bf --- /dev/null +++ b/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/conf/IcebergConf.java @@ -0,0 +1,69 @@ +package com.dtstack.chunjun.connector.iceberg.conf; + +import com.dtstack.chunjun.conf.ChunJunCommonConf; +import com.dtstack.chunjun.sink.WriteMode; + +import java.util.HashMap; +import java.util.Map; + +public class IcebergConf extends ChunJunCommonConf { + private String warehouse; + + private String uri; + + private Map hadoopConfig = new HashMap<>(); + + private String database; + + private String table; + + private String writeMode = WriteMode.APPEND.name(); + + public void setDatabase(String database) { + this.database = database; + } + + public String getDatabase() { + return database; + } + + public void setTable(String table) { + this.table = table; + } + + public String getTable() { + return table; + } + + public void setWarehouse(String warehouse) { + this.warehouse = warehouse; + } + + public String getWarehouse() { + return warehouse; + } + + public void setUri(String uri) { + this.uri = uri; + } + + public String getUri() { + return uri; + } + + public void setHadoopConfig(Map hadoopConfig) { + this.hadoopConfig = hadoopConfig; + } + + public Map getHadoopConfig() { + return hadoopConfig; + } + + public String getWriteMode() { + return writeMode; + } + + public void setWriteMode(String writeMode) { + this.writeMode = writeMode; + } +} diff --git a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/conf/IcebergReaderConf.java b/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/conf/IcebergReaderConf.java deleted file mode 100644 index c12dfb6299..0000000000 --- a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/conf/IcebergReaderConf.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.dtstack.chunjun.connector.iceberg.conf; - -import com.dtstack.chunjun.conf.ChunJunCommonConf; - -import java.util.HashMap; -import java.util.Map; - -public class IcebergReaderConf extends ChunJunCommonConf { - private String path; - - private String defaultFS; - private String fileType; - /** hadoop高可用相关配置 * */ - private Map hadoopConfig = new HashMap<>(16); - - public String getDefaultFS() { - return defaultFS; - } - - public void setDefaultFS(String defaultFS) { - this.defaultFS = defaultFS; - } - - public String getFileType() { - return fileType; - } - - public void setFileType(String fileType) { - this.fileType = fileType; - } - - public Map getHadoopConfig() { - return hadoopConfig; - } - - public void setHadoopConfig(Map hadoopConfig) { - this.hadoopConfig = hadoopConfig; - } - - public String getPath() { - return path; - } - - public void setPath(String path) { - this.path = path; - } - - @Override - public String toString() { - return "IcebergReaderConf{" + "path='" + path + '\'' + '}'; - } -} diff --git a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/conf/IcebergWriterConf.java b/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/conf/IcebergWriterConf.java deleted file mode 100644 index c610591cb5..0000000000 --- a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/conf/IcebergWriterConf.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.chunjun.connector.iceberg.conf; - -import com.dtstack.chunjun.conf.ChunJunCommonConf; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class IcebergWriterConf extends ChunJunCommonConf { - private String defaultFS; - private String fileType; - /** hadoop高可用相关配置 * */ - private Map hadoopConfig = new HashMap<>(16); - - private String path; - - private String writeMode; - /** upsert模式需要传入主键列表 */ - private List primaryKey = new ArrayList<>(); - - public static final String APPEND_WRITE_MODE = "append"; - public static final String UPSERT_WRITE_MODE = "upsert"; - public static final String OVERWRITE_WRITE_MODE = "overwrite"; - - public String getDefaultFS() { - return defaultFS; - } - - public void setDefaultFS(String defaultFS) { - this.defaultFS = defaultFS; - } - - public String getFileType() { - return fileType; - } - - public void setFileType(String fileType) { - this.fileType = fileType; - } - - public Map getHadoopConfig() { - return hadoopConfig; - } - - public void setHadoopConfig(Map hadoopConfig) { - this.hadoopConfig = hadoopConfig; - } - - public List getPrimaryKey() { - return primaryKey; - } - - public void setPrimaryKey(List primaryKey) { - this.primaryKey = primaryKey; - } - - public String getWriteMode() { - return writeMode; - } - - public void setWriteMode(String writeMode) { - this.writeMode = writeMode; - } - - public String getPath() { - return path; - } - - public void setPath(String path) { - this.path = path; - } - - @Override - public String toString() { - return "IcebergWriterConf{" + "path='" + path + '\'' + '}'; - } -} diff --git a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/converter/IcebergColumnConverter.java b/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/converter/IcebergColumnConverter.java deleted file mode 100644 index 1fee587d9f..0000000000 --- a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/converter/IcebergColumnConverter.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.dtstack.chunjun.connector.iceberg.converter; - -import com.dtstack.chunjun.conf.FieldConf; -import com.dtstack.chunjun.constants.ConstantValue; -import com.dtstack.chunjun.converter.AbstractRowConverter; -import com.dtstack.chunjun.converter.IDeserializationConverter; -import com.dtstack.chunjun.converter.ISerializationConverter; -import com.dtstack.chunjun.element.AbstractBaseColumn; -import com.dtstack.chunjun.element.ColumnRowData; -import com.dtstack.chunjun.element.column.BigDecimalColumn; -import com.dtstack.chunjun.element.column.BooleanColumn; -import com.dtstack.chunjun.element.column.BytesColumn; -import com.dtstack.chunjun.element.column.StringColumn; -import com.dtstack.chunjun.element.column.TimeColumn; -import com.dtstack.chunjun.element.column.TimestampColumn; -import com.dtstack.chunjun.throwable.ChunJunRuntimeException; -import com.dtstack.chunjun.throwable.UnsupportedTypeException; -import com.dtstack.chunjun.util.ColumnTypeUtil; - -import org.apache.flink.table.data.DecimalData; -import org.apache.flink.table.data.GenericRowData; -import org.apache.flink.table.data.RowData; -import org.apache.flink.table.data.TimestampData; -import org.apache.flink.table.data.binary.BinaryStringData; - -import java.util.List; -import java.util.Locale; -import java.util.Map; - -public class IcebergColumnConverter extends AbstractRowConverter { - - private List columnNameList; - private transient Map decimalColInfo; - - public IcebergColumnConverter(List fieldConfList) { - super(fieldConfList.size()); - for (int i = 0; i < fieldConfList.size(); i++) { - String type = fieldConfList.get(i).getType(); - int left = type.indexOf(ConstantValue.LEFT_PARENTHESIS_SYMBOL); - int right = type.indexOf(ConstantValue.RIGHT_PARENTHESIS_SYMBOL); - if (left > 0 && right > 0) { - type = type.substring(0, left); - } - toInternalConverters.add( - wrapIntoNullableInternalConverter(createInternalConverter(type))); - // toExternalConverters.add( - // wrapIntoNullableExternalConverter(createExternalConverter(type), - // type)); - } - } - - @Override - @SuppressWarnings("unchecked") - public RowData toInternal(RowData input) throws Exception { - ColumnRowData row = new ColumnRowData(input.getArity()); - if (input instanceof GenericRowData) { - GenericRowData genericRowData = (GenericRowData) input; - for (int i = 0; i < input.getArity(); i++) { - Object field = genericRowData.getField(i); - AbstractBaseColumn abstractBaseColumn = - (AbstractBaseColumn) toInternalConverters.get(i).deserialize(field); - row.addField(abstractBaseColumn); - } - } else { - throw new ChunJunRuntimeException( - "Error RowData type, RowData:[" - + input - + "] should be instance of GenericRowData."); - } - return row; - } - - @Override - @SuppressWarnings("unchecked") - public String toExternal(RowData rowData, String group) throws Exception { - for (int index = 0; index < rowData.getArity(); index++) { - toExternalConverters.get(index).serialize(rowData, index, group); - } - return group; - } - - @Override - public RowData toInternalLookup(RowData input) { - throw new ChunJunRuntimeException("HDFS Connector doesn't support Lookup Table Function."); - } - - @Override - @SuppressWarnings("unchecked") - protected ISerializationConverter wrapIntoNullableExternalConverter( - ISerializationConverter serializationConverter, String type) { - return (rowData, index, group) -> { - if (rowData == null || rowData.isNullAt(index)) { - // do nothing - } else { - serializationConverter.serialize(rowData, index, group); - } - }; - } - - /** 根据json中字段的类型转成chunjun sync里的类型 ：参考 com.dtstack.chunjun.element.AbstractBaseColumn */ - @Override - @SuppressWarnings("all") - protected IDeserializationConverter createInternalConverter(String type) { - switch (type.toUpperCase(Locale.ENGLISH)) { - case "BOOLEAN": - return (IDeserializationConverter) BooleanColumn::new; - case "TINYINT": - case "SMALLINT": - case "INT": - return (IDeserializationConverter) - BigDecimalColumn::new; - case "BIGINT": - case "LONG": - return (IDeserializationConverter) BigDecimalColumn::new; - case "FLOAT": - return (IDeserializationConverter) BigDecimalColumn::new; - case "DOUBLE": - return (IDeserializationConverter) - BigDecimalColumn::new; - case "DECIMAL": - return (IDeserializationConverter) - val -> new BigDecimalColumn(val.toBigDecimal()); - case "STRING": - case "VARCHAR": - case "CHAR": - return (IDeserializationConverter) - val -> new StringColumn(val.toString()); - case "TIMESTAMP": - return (IDeserializationConverter) - val -> new TimestampColumn(val.toTimestamp()); - case "TIME": - return (IDeserializationConverter) - // todo Iceberg里存储time实际是存储成一个integer类型，如63923598代表18:08:51.871 - val -> new TimeColumn(val); - case "DATE": - return (IDeserializationConverter) - // todo Iceberg里存储date实际是存储成一个integer类型，如19235 代表2022-08-31 - val -> new TimestampColumn(val); - case "BINARY": - return (IDeserializationConverter) BytesColumn::new; - case "ARRAY": - case "MAP": - case "STRUCT": - case "UNION": - default: - throw new UnsupportedTypeException(type); - } - } - - @Override - protected ISerializationConverter createExternalConverter(String type) { - throw new ChunJunRuntimeException("Sink type conversion is not supported! "); - } - - public void setColumnNameList(List columnNameList) { - this.columnNameList = columnNameList; - } - - public void setDecimalColInfo(Map decimalColInfo) { - this.decimalColInfo = decimalColInfo; - } -} diff --git a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/converter/IcebergRawTypeConverter.java b/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/converter/IcebergRawTypeConverter.java deleted file mode 100644 index a4022c74db..0000000000 --- a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/converter/IcebergRawTypeConverter.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.chunjun.connector.iceberg.converter; - -import com.dtstack.chunjun.throwable.UnsupportedTypeException; - -import org.apache.flink.table.api.DataTypes; -import org.apache.flink.table.types.DataType; - -import java.util.Locale; - -public class IcebergRawTypeConverter { - - /** 将Iceberg中的类型，转换成flink的DataType类型。。 */ - public static DataType apply(String type) { - switch (type.toUpperCase(Locale.ENGLISH)) { - case "INT": - case "INTEGER": - return DataTypes.INT(); - case "BIGINT": - case "LONG": - return DataTypes.BIGINT(); - case "BOOLEAN": - return DataTypes.BOOLEAN(); - case "FLOAT": - return DataTypes.FLOAT(); - case "DECIMAL": - case "NUMERIC": - return DataTypes.DECIMAL(1, 0); - case "DOUBLE": - return DataTypes.DOUBLE(); - case "CHAR": - case "VARCHAR": - case "STRING": - return DataTypes.STRING(); - case "DATE": - return DataTypes.DATE(); - case "TIME": - return DataTypes.TIME(); - case "TIMESTAMP": - case "DATETIME": - return DataTypes.TIMESTAMP(); - default: - throw new UnsupportedTypeException(type); - } - } -} diff --git a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/sink/ChunjunRowDataConvertMap.java b/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/sink/ChunjunRowDataConvertMap.java new file mode 100644 index 0000000000..e6e7c344c9 --- /dev/null +++ b/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/sink/ChunjunRowDataConvertMap.java @@ -0,0 +1,98 @@ +package com.dtstack.chunjun.connector.iceberg.sink; + +import com.dtstack.chunjun.conf.FieldConf; +import com.dtstack.chunjun.element.ColumnRowData; +import com.dtstack.chunjun.throwable.UnsupportedTypeException; + +import org.apache.flink.api.common.functions.MapFunction; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.types.RowKind; + +import java.util.List; +import java.util.Locale; + +public class ChunjunRowDataConvertMap implements MapFunction { + private List columns; + + public ChunjunRowDataConvertMap(List columns) { + this.columns = columns; + } + + @Override + public RowData map(RowData row) { + if (row instanceof ColumnRowData) { + GenericRowData convertedData = new GenericRowData(RowKind.INSERT, columns.size()); + + /** 只有数据还原才有headers scn, schema, table, ts, opTime, type, before, after */ + boolean hasHeader = ((ColumnRowData) row).getHeaders() != null; + + for (FieldConf column : columns) { + int index = 0; + if (hasHeader) { + index = 7 + column.getIndex(); + } else { + index = column.getIndex(); + } + String type = column.getType(); + Object value = getRowDataByType(row, type, index); + convertedData.setField(column.getIndex(), value); + } + + return convertedData; + } + return row; + } + + private Object getRowDataByType(RowData data, String type, int index) { + switch (type.toUpperCase(Locale.ENGLISH)) { + case "BOOLEAN": + case "BIT": + return data.getBoolean(index); + case "TINYINT": + case "TINYINT UNSIGNED": + case "SMALLINT": + case "SMALLINT UNSIGNED": + case "MEDIUMINT": + case "MEDIUMINT UNSIGNED": + case "INT": + case "INTEGER": + case "INT24": + case "INT UNSIGNED": + return data.getInt(index); + case "BIGINT": + case "BIGINT UNSIGNED": + return data.getLong(index); + case "DECIMAL": + case "DECIMAL UNSIGNED": + case "NUMERIC": + return data.getDecimal(index, 14, 2); + case "REAL": + case "FLOAT": + case "FLOAT UNSIGNED": + return data.getFloat(index); + case "DOUBLE": + case "DOUBLE UNSIGNED": + return data.getDouble(index); + case "CHAR": + case "VARCHAR": + case "STRING": + case "TINYTEXT": + case "TEXT": + case "MEDIUMTEXT": + case "LONGTEXT": + case "JSON": + case "ENUM": + case "SET": + return data.getString(index); + case "DATE": + case "TIME": + case "TIMESTAMP": + case "DATETIME": + return data.getTimestamp(index, 0); + + default: + throw new UnsupportedTypeException(type); + } + } +} diff --git a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/sink/IcebergMetricsMapFunction.java b/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/sink/IcebergMetricsMapFunction.java deleted file mode 100644 index 39afba5d13..0000000000 --- a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/sink/IcebergMetricsMapFunction.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.chunjun.connector.iceberg.sink; - -import com.dtstack.chunjun.conf.ChunJunCommonConf; -import com.dtstack.chunjun.constants.Metrics; -import com.dtstack.chunjun.metrics.AccumulatorCollector; -import com.dtstack.chunjun.metrics.BaseMetric; -import com.dtstack.chunjun.metrics.RowSizeCalculator; - -import org.apache.flink.api.common.accumulators.LongCounter; -import org.apache.flink.api.common.functions.RichMapFunction; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; -import org.apache.flink.table.data.RowData; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Map; - -public class IcebergMetricsMapFunction extends RichMapFunction { - protected final Logger LOG = LoggerFactory.getLogger(getClass()); - - /** 输出指标组 */ - protected transient BaseMetric outputMetric; - /** 环境上下文 */ - protected StreamingRuntimeContext context; - /** 累加器收集器 */ - protected AccumulatorCollector accumulatorCollector; - /** 对象大小计算器 */ - protected RowSizeCalculator rowSizeCalculator; - - protected LongCounter bytesWriteCounter; - protected LongCounter durationCounter; - protected LongCounter numWriteCounter; - protected LongCounter snapshotWriteCounter; - protected LongCounter errCounter; - protected LongCounter nullErrCounter; - protected LongCounter duplicateErrCounter; - protected LongCounter conversionErrCounter; - protected LongCounter otherErrCounter; - - /** 任务开始时间, openInputFormat()开始计算 */ - protected long startTime; - /** 任务公共配置 */ - protected ChunJunCommonConf config; - - /** 任务名称 */ - protected String jobName = "defaultJobName"; - /** 任务id */ - protected String jobId; - /** 任务索引id */ - protected int taskNumber; - /** 子任务数量 */ - protected int numTasks; - - public IcebergMetricsMapFunction(ChunJunCommonConf config) { - this.config = config; - } - - @Override - public void open(Configuration parameters) throws Exception { - super.open(parameters); - context = (StreamingRuntimeContext) getRuntimeContext(); - numTasks = context.getNumberOfParallelSubtasks(); - taskNumber = context.getIndexOfThisSubtask(); - this.startTime = System.currentTimeMillis(); - - Map vars = context.getMetricGroup().getAllVariables(); - if (vars != null) { - jobName = vars.getOrDefault(Metrics.JOB_NAME, "defaultJobName"); - jobId = vars.get(Metrics.JOB_ID); - } - - initStatisticsAccumulator(); - initAccumulatorCollector(); - initRowSizeCalculator(); - } - - @Override - public RowData map(RowData rowData) throws Exception { - numWriteCounter.add(1L); - updateDuration(); - bytesWriteCounter.add(rowSizeCalculator.getObjectSize(rowData)); - return rowData; - } - - @Override - public void close() throws Exception { - updateDuration(); - if (outputMetric != null) { - outputMetric.waitForReportMetrics(); - } - if (accumulatorCollector != null) { - accumulatorCollector.close(); - } - LOG.info("subtask[{}}] close() finished", taskNumber); - - super.close(); - } - - /** 初始化累加器指标 */ - protected void initStatisticsAccumulator() { - errCounter = context.getLongCounter(Metrics.NUM_ERRORS); - nullErrCounter = context.getLongCounter(Metrics.NUM_NULL_ERRORS); - duplicateErrCounter = context.getLongCounter(Metrics.NUM_DUPLICATE_ERRORS); - conversionErrCounter = context.getLongCounter(Metrics.NUM_CONVERSION_ERRORS); - otherErrCounter = context.getLongCounter(Metrics.NUM_OTHER_ERRORS); - numWriteCounter = context.getLongCounter(Metrics.NUM_WRITES); - snapshotWriteCounter = context.getLongCounter(Metrics.SNAPSHOT_WRITES); - bytesWriteCounter = context.getLongCounter(Metrics.WRITE_BYTES); - durationCounter = context.getLongCounter(Metrics.WRITE_DURATION); - - outputMetric = new BaseMetric(context); - outputMetric.addMetric(Metrics.NUM_ERRORS, errCounter); - outputMetric.addMetric(Metrics.NUM_NULL_ERRORS, nullErrCounter); - outputMetric.addMetric(Metrics.NUM_DUPLICATE_ERRORS, duplicateErrCounter); - outputMetric.addMetric(Metrics.NUM_CONVERSION_ERRORS, conversionErrCounter); - outputMetric.addMetric(Metrics.NUM_OTHER_ERRORS, otherErrCounter); - outputMetric.addMetric(Metrics.NUM_WRITES, numWriteCounter, true); - outputMetric.addMetric(Metrics.SNAPSHOT_WRITES, snapshotWriteCounter); - outputMetric.addMetric(Metrics.WRITE_BYTES, bytesWriteCounter, true); - outputMetric.addMetric(Metrics.WRITE_DURATION, durationCounter); - // outputMetric.addDirtyMetric( - // Metrics.DIRTY_DATA_COUNT, this.dirtyManager.getConsumedMetric()); - // outputMetric.addDirtyMetric( - // Metrics.DIRTY_DATA_COLLECT_FAILED_COUNT, - // this.dirtyManager.getFailedConsumedMetric()); - } - - /** 初始化累加器收集器 */ - private void initAccumulatorCollector() { - accumulatorCollector = new AccumulatorCollector(context, Metrics.METRIC_SINK_LIST); - accumulatorCollector.start(); - } - - /** 更新任务执行时间指标 */ - protected void updateDuration() { - if (durationCounter != null) { - durationCounter.resetLocal(); - durationCounter.add(System.currentTimeMillis() - startTime); - } - } - - /** 初始化对象大小计算器 */ - protected void initRowSizeCalculator() { - rowSizeCalculator = - RowSizeCalculator.getRowSizeCalculator(config.getRowSizeCalculatorType(), true); - } -} diff --git a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/sink/IcebergSinkFactory.java b/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/sink/IcebergSinkFactory.java index 2ad18429bd..0cee599e16 100644 --- a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/sink/IcebergSinkFactory.java +++ b/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/sink/IcebergSinkFactory.java @@ -1,114 +1,85 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package com.dtstack.chunjun.connector.iceberg.sink; import com.dtstack.chunjun.conf.SyncConf; -import com.dtstack.chunjun.connector.iceberg.conf.IcebergWriterConf; +import com.dtstack.chunjun.connector.iceberg.conf.IcebergConf; import com.dtstack.chunjun.converter.RawTypeConverter; import com.dtstack.chunjun.sink.SinkFactory; -import com.dtstack.chunjun.util.FileSystemUtil; -import com.dtstack.chunjun.util.GsonUtil; +import com.dtstack.chunjun.sink.WriteMode; +import com.dtstack.chunjun.util.JsonUtil; -import org.apache.flink.api.common.io.OutputFormat; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.DataStreamSink; -import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.table.data.RowData; -import org.apache.flink.util.Preconditions; -import com.google.common.collect.Lists; import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.flink.CatalogLoader; import org.apache.iceberg.flink.TableLoader; import org.apache.iceberg.flink.sink.FlinkSink; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + public class IcebergSinkFactory extends SinkFactory { - private final IcebergWriterConf writerConf; + private IcebergConf icebergConf; - public IcebergSinkFactory(SyncConf config) { - super(config); - writerConf = - GsonUtil.GSON.fromJson( - GsonUtil.GSON.toJson(config.getWriter().getParameter()), - IcebergWriterConf.class); - writerConf.setColumn(config.getWriter().getFieldList()); - super.initCommonConf(writerConf); + public IcebergSinkFactory(SyncConf conf) { + super(conf); + icebergConf = + JsonUtil.toObject( + JsonUtil.toJson(syncConf.getWriter().getParameter()), IcebergConf.class); } @Override - public DataStreamSink createSink(DataStream dataSet) { - if (!useAbstractBaseColumn) { - throw new UnsupportedOperationException("iceberg not support transform"); - } - return createOutput(dataSet, null); + public RawTypeConverter getRawTypeConverter() { + return null; } @Override - protected DataStreamSink createOutput( - DataStream dataSet, OutputFormat outputFormat) { - return createOutput(dataSet, outputFormat, this.getClass().getSimpleName().toLowerCase()); - } + public DataStreamSink createSink(DataStream dataSet) { + List columns = + icebergConf.getColumn().stream() + .map(col -> col.getName()) + .collect(Collectors.toList()); - @Override - protected DataStreamSink createOutput( - DataStream dataSet, OutputFormat outputFormat, String sinkName) { - Preconditions.checkNotNull(dataSet); - Preconditions.checkNotNull(sinkName); - - // 初始化 hadoop conf - Configuration conf = - FileSystemUtil.getConfiguration( - writerConf.getHadoopConfig(), writerConf.getDefaultFS()); - - TableLoader tableLoader = TableLoader.fromHadoopTable(writerConf.getPath(), conf); - SingleOutputStreamOperator streamOperator = - dataSet.map(new IcebergMetricsMapFunction(writerConf)); - DataStreamSink dataDataStreamSink = null; - // 判断写出模式 - String writeMode = writerConf.getWriteMode(); - if (writeMode.equals(IcebergWriterConf.UPSERT_WRITE_MODE)) { - dataDataStreamSink = - FlinkSink.forRowData(streamOperator) - .tableLoader(tableLoader) - .equalityFieldColumns(Lists.newArrayList("id")) - .upsert(true) - .build(); - } else if (writeMode.equals(IcebergWriterConf.OVERWRITE_WRITE_MODE)) { - dataDataStreamSink = - FlinkSink.forRowData(streamOperator) - .tableLoader(tableLoader) - .overwrite(true) - .build(); - - } else if (writeMode.equals(IcebergWriterConf.APPEND_WRITE_MODE)) { - dataDataStreamSink = - FlinkSink.forRowData(streamOperator).tableLoader(tableLoader).build(); - } else { - throw new UnsupportedOperationException("iceberg not support writeMode :" + writeMode); - } + DataStream convertedDataStream = + dataSet.map(new ChunjunRowDataConvertMap(icebergConf.getColumn())); - dataDataStreamSink.name(sinkName); - return dataDataStreamSink; + boolean isOverwrite = + icebergConf.getWriteMode().equalsIgnoreCase(WriteMode.OVERWRITE.name()); + return FlinkSink.forRowData(convertedDataStream) + .tableLoader(buildTableLoader()) + .writeParallelism(icebergConf.getParallelism()) + .equalityFieldColumns(columns) + .overwrite(isOverwrite) + .build(); } - @Override - public RawTypeConverter getRawTypeConverter() { - return null; + private TableLoader buildTableLoader() { + Map icebergProps = new HashMap<>(); + icebergProps.put("warehouse", icebergConf.getWarehouse()); + icebergProps.put("uri", icebergConf.getUri()); + + /* build hadoop configuration */ + Configuration configuration = new Configuration(); + icebergConf + .getHadoopConfig() + .entrySet() + .forEach(kv -> configuration.set(kv.getKey(), (String) kv.getValue())); + + CatalogLoader hc = + CatalogLoader.hive(icebergConf.getDatabase(), configuration, icebergProps); + TableLoader tl = + TableLoader.fromCatalog( + hc, TableIdentifier.of(icebergConf.getDatabase(), icebergConf.getTable())); + + if (tl instanceof TableLoader.CatalogTableLoader) { + tl.open(); + } + + return tl; } } diff --git a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/source/IcebergInputFormat.java b/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/source/IcebergInputFormat.java deleted file mode 100644 index 28ed95e995..0000000000 --- a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/source/IcebergInputFormat.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.chunjun.connector.iceberg.source; - -import com.dtstack.chunjun.connector.iceberg.conf.IcebergReaderConf; -import com.dtstack.chunjun.source.format.BaseRichInputFormat; -import com.dtstack.chunjun.throwable.ReadRecordException; - -import org.apache.flink.core.io.InputSplit; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.table.data.RowData; - -import org.apache.iceberg.flink.source.FlinkInputFormat; -import org.apache.iceberg.flink.source.FlinkInputSplit; - -import java.io.IOException; - -public class IcebergInputFormat extends BaseRichInputFormat { - private FlinkInputFormat flinkInputFormat; - private IcebergReaderConf icebergReaderConf; - private StreamExecutionEnvironment env; - - public IcebergInputFormat() {} - - public void setInput(FlinkInputFormat input) { - this.flinkInputFormat = input; - } - - public void setIcebergReaderConf(IcebergReaderConf icebergReaderConf) { - this.icebergReaderConf = icebergReaderConf; - } - - @Override - public void openInputFormat() throws IOException { - super.openInputFormat(); - } - - @Override - protected InputSplit[] createInputSplitsInternal(int minNumSplits) throws Exception { - return flinkInputFormat.createInputSplits(minNumSplits); - } - - @Override - protected void openInternal(InputSplit inputSplit) throws IOException { - flinkInputFormat.open((FlinkInputSplit) inputSplit); - } - - @Override - protected RowData nextRecordInternal(RowData rowData) throws ReadRecordException { - RowData genericRowData = flinkInputFormat.nextRecord(rowData); - try { - RowData columnRowData = rowConverter.toInternal(genericRowData); - return columnRowData; - } catch (Exception e) { - throw new ReadRecordException("", e, 0, rowData); - } - } - - @Override - protected void closeInternal() throws IOException { - flinkInputFormat.close(); - } - - @Override - public boolean reachedEnd() throws IOException { - return flinkInputFormat.reachedEnd(); - } -} diff --git a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/source/IcebergInputFormatBuilder.java b/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/source/IcebergInputFormatBuilder.java deleted file mode 100644 index 1844a444f1..0000000000 --- a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/source/IcebergInputFormatBuilder.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.chunjun.connector.iceberg.source; - -import com.dtstack.chunjun.connector.iceberg.conf.IcebergReaderConf; -import com.dtstack.chunjun.source.format.BaseRichInputFormatBuilder; - -import org.apache.commons.collections.CollectionUtils; -import org.apache.iceberg.flink.source.FlinkInputFormat; - -public class IcebergInputFormatBuilder extends BaseRichInputFormatBuilder { - - public IcebergInputFormatBuilder() { - super(new IcebergInputFormat()); - } - - public void setIcebergConf(IcebergReaderConf icebergConf) { - super.setConfig(icebergConf); - format.setIcebergReaderConf(icebergConf); - } - - public void setInput(FlinkInputFormat input) { - format.setInput(input); - } - - @Override - protected void checkFormat() { - if (CollectionUtils.isEmpty(format.getConfig().getColumn())) { - throw new IllegalArgumentException("columns can not be empty"); - } - } -} diff --git a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/source/IcebergSourceFactory.java b/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/source/IcebergSourceFactory.java deleted file mode 100644 index 87146921ba..0000000000 --- a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/source/IcebergSourceFactory.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.chunjun.connector.iceberg.source; - -import com.dtstack.chunjun.conf.SyncConf; -import com.dtstack.chunjun.connector.iceberg.conf.IcebergReaderConf; -import com.dtstack.chunjun.connector.iceberg.converter.IcebergColumnConverter; -import com.dtstack.chunjun.connector.iceberg.converter.IcebergRawTypeConverter; -import com.dtstack.chunjun.converter.AbstractRowConverter; -import com.dtstack.chunjun.converter.RawTypeConverter; -import com.dtstack.chunjun.source.SourceFactory; -import com.dtstack.chunjun.util.FileSystemUtil; -import com.dtstack.chunjun.util.GsonUtil; - -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.table.data.RowData; - -import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.flink.TableLoader; -import org.apache.iceberg.flink.source.FlinkInputFormat; -import org.apache.iceberg.flink.source.FlinkSource; - -public class IcebergSourceFactory extends SourceFactory { - - private final IcebergReaderConf icebergConf; - - public IcebergSourceFactory(SyncConf config, StreamExecutionEnvironment env) { - super(config, env); - icebergConf = - GsonUtil.GSON.fromJson( - GsonUtil.GSON.toJson(config.getReader().getParameter()), - IcebergReaderConf.class); - icebergConf.setColumn(config.getReader().getFieldList()); - super.initCommonConf(icebergConf); - } - - @Override - public DataStream createSource() { - IcebergInputFormatBuilder builder = new IcebergInputFormatBuilder(); - builder.setIcebergConf(icebergConf); - // 初始化 hadoop conf - Configuration conf = - FileSystemUtil.getConfiguration( - icebergConf.getHadoopConfig(), icebergConf.getDefaultFS()); - - TableLoader tableLoader = TableLoader.fromHadoopTable(icebergConf.getPath(), conf); - FlinkInputFormat flinkInputFormat = - FlinkSource.forRowData().env(env).tableLoader(tableLoader).buildFormat(); - builder.setInput(flinkInputFormat); - - AbstractRowConverter rowConverter; - // if (useAbstractBaseColumn) { - rowConverter = new IcebergColumnConverter(icebergConf.getColumn()); - // } else { - // checkConstant(icebergConf); - // final RowType rowType = - // TableUtil.createRowType(icebergConf.getColumn(), - // getRawTypeConverter()); - // rowConverter = new StreamRowConverter(rowType); - // } - builder.setRowConverter(rowConverter, useAbstractBaseColumn); - - return createInput(builder.finish()); - } - - @Override - public RawTypeConverter getRawTypeConverter() { - return IcebergRawTypeConverter::apply; - } -} diff --git a/chunjun-examples/json/iceberg/iceberg_stream.json b/chunjun-examples/json/iceberg/iceberg_stream.json deleted file mode 100644 index 210d6d8835..0000000000 --- a/chunjun-examples/json/iceberg/iceberg_stream.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "job": { - "content": [ - { - "reader" : { - "parameter" : { - "path" : "/Users/xx/study/bigdata-iceberg/iceberg_warehouse/tb1", - "defaultFS" : "file:///", - "hadoopConfig" : { - "fs.hdfs.impl.disable.cache": "true" - }, - "column": [ - { - "name": "id", - "type": "int" - }, - { - "name": "event_time", - "type": "long" - }, - { - "name": "message", - "type": "string" - }, - { - "name": "t_timestamp", - "type": "timestamp" - }, - { - "name": "t_date", - "type": "date" - }, - { - "name": "t_time", - "type": "time" - }, - { - "name": "t_bool", - "type": "boolean" - }, - { - "name": "t_double", - "type": "double" - }, - { - "name": "t_float", - "type": "float" - }, - { - "name": "t_decimal", - "type": "decimal" - } - ] - }, - "name" : "icebergreader" - }, - "writer": { - "name": "streamwriter", - "parameter": { - "print": true - } - } - } - ], - "setting": { - "speed": { - "channel": 1, - "bytes": 0 - } - } - } -} diff --git a/chunjun-examples/json/iceberg/stream_iceberg.json b/chunjun-examples/json/iceberg/stream_iceberg.json index 5c62747e30..22bdc08425 100644 --- a/chunjun-examples/json/iceberg/stream_iceberg.json +++ b/chunjun-examples/json/iceberg/stream_iceberg.json @@ -8,14 +8,6 @@ { "name": "id", "type": "id" - }, - { - "name": "sell", - "type": "long" - }, - { - "name": "content", - "type": "string" } ], "sliceRecordCount": ["10"], @@ -25,17 +17,29 @@ }, "writer": { "parameter": { - "path":"/Users/xx/study/bigdata-iceberg/iceberg_warehouse/tb2", - "writeMode": "upsert", - "primaryKey": ["id"], - "defaultFS" : "file:///", - "hadoopConfig" : { - "fs.hdfs.impl.disable.cache": "true" - } + "column": [ { + "name": "id", + "index": 0, + "resourceName": "", + "type": "INT", + "key": "id" + }], + "uri": "thrift://172-16-23-238:9083", + "warehouse": "hdfs:///dtInsight/hive/warehouse", + "database": "kungen", + "table": "test1234", + "hadoopConfig": { + "fs.defaultFS":"hdfs://ns1", + "dfs.nameservices":"ns1", + "dfs.ha.namenodes.ns1":"nn1,nn2", + "dfs.namenode.rpc-address.ns1.nn1":"172.16.21.107:9000", + "dfs.namenode.rpc-address.ns1.nn2":"172.16.22.103:9000", + "dfs.client.failover.proxy.provider.ns1": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider" + }, + "writeMode": "overwrite" }, "name": "icebergwriter" } - } ], "setting": { diff --git a/chunjun-local-test/pom.xml b/chunjun-local-test/pom.xml index f21c1fdd4d..40e01a5ec9 100644 --- a/chunjun-local-test/pom.xml +++ b/chunjun-local-test/pom.xml @@ -142,6 +142,12 @@ ${project.version} + + com.dtstack.chunjun + chunjun-connector-iceberg + ${project.version} + + com.dtstack.chunjun diff --git "a/docs_zh/ChunJun\350\277\236\346\216\245\345\231\250/iceberg/iceberg-sink.md" "b/docs_zh/ChunJun\350\277\236\346\216\245\345\231\250/iceberg/iceberg-sink.md" new file mode 100644 index 0000000000..2a153b172f --- /dev/null +++ "b/docs_zh/ChunJun\350\277\236\346\216\245\345\231\250/iceberg/iceberg-sink.md" @@ -0,0 +1,45 @@ +# Iceberg Sink + +## 一、介绍 +将数据写入数据湖iceberg中, 目前只支持追加模式 + + +## 二、参数说明 +| 参数名 | 是否必填 | 默认值 | 类型 | 含义 | +| ------------ | -------- | ------ | ------ | ---------------------------------------------------- | +| warehouse | 是 | none | String | hive的路径，示例: (hdfs:///dtInsight/hive/warehouse) | +| uri | 是 | none | String | metastore uri | +| hadoopConfig | 是 | none | map | hadoop配置 | +| database | 是 | none | String | iceberg 数据库 | +| table | 是 | none | String | iceberg 表名 | +| writeMode | 否 | append | String | 写入模式, 支持overwrite + +示例: +```json +{ + "parameter": { + "column": [ { + "name": "id", + "index": 0, + "resourceName": "", + "type": "INT", + "key": "id" + }], + "uri": "thrift://172-16-23-238:9083", + "warehouse": "hdfs:///dtInsight/hive/warehouse", + "database": "kungen", + "table": "test1234", + "hadoopConfig": { + "fs.defaultFS":"hdfs://ns1", + "dfs.nameservices":"ns1", + "dfs.ha.namenodes.ns1":"nn1,nn2", + "dfs.namenode.rpc-address.ns1.nn1":"172.16.21.107:9000", + "dfs.namenode.rpc-address.ns1.nn2":"172.16.22.103:9000", + "dfs.client.failover.proxy.provider.ns1": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider" + }, + "writeMode": "overwrite" + }, + "name": "icebergwriter" +} +``` + From 738a0336f51909de37038a3b5a2c5a25c68a725b Mon Sep 17 00:00:00 2001 From: yanghuaiGit <38883656+yanghuaiGit@users.noreply.github.com> Date: Mon, 6 Feb 2023 16:43:38 +0800 Subject: [PATCH 29/41] #1494 [feat-1494][http] http support array data (#1495) (cherry picked from commit 58fbb3b79e9aca5b50864ac61a20d4ebdfd5aba8) --- .../chunjun-connector-http/pom.xml | 15 ++ .../http/client/CsvResponseParse.java | 84 ++++++++++ .../http/client/DefaultRestHandler.java | 6 - .../connector/http/client/HttpClient.java | 57 +++++-- .../http/client/JsonResponseParse.java | 121 ++++++++++++++ .../connector/http/client/ResponseParse.java | 95 +++++++++++ .../connector/http/client/ResponseValue.java | 10 +- .../connector/http/client/RestHandler.java | 11 -- .../http/client/TextResponseParse.java | 58 +++++++ .../http/client/XmlResponseParse.java | 98 +++++++++++ .../connector/http/common/ConstantValue.java | 4 + .../connector/http/common/HttpRestConfig.java | 78 ++++++++- .../connector/http/common/HttpUtil.java | 17 +- .../http/converter/HttpColumnConverter.java | 47 +++--- .../http/inputformat/HttpInputFormat.java | 13 +- .../connector/http/util/JsonPathUtil.java | 40 +++++ .../chunjun/connector/http/util/XmlUtil.java | 156 ++++++++++++++++++ 17 files changed, 834 insertions(+), 76 deletions(-) create mode 100644 chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/CsvResponseParse.java create mode 100644 chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/JsonResponseParse.java create mode 100644 chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/ResponseParse.java create mode 100644 chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/TextResponseParse.java create mode 100644 chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/XmlResponseParse.java create mode 100644 chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/util/JsonPathUtil.java create mode 100644 chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/util/XmlUtil.java diff --git a/chunjun-connectors/chunjun-connector-http/pom.xml b/chunjun-connectors/chunjun-connector-http/pom.xml index 3a32617257..f1724cdcf1 100644 --- a/chunjun-connectors/chunjun-connector-http/pom.xml +++ b/chunjun-connectors/chunjun-connector-http/pom.xml @@ -31,6 +31,21 @@ chunjun-connector-http ChunJun : Connectors : Http + + + + org.dom4j + dom4j + 2.1.3 + + + + net.sourceforge.javacsv + javacsv + 2.0 + + + diff --git a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/CsvResponseParse.java b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/CsvResponseParse.java new file mode 100644 index 0000000000..08beb1887b --- /dev/null +++ b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/CsvResponseParse.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.connector.http.client; + +import com.dtstack.chunjun.connector.http.common.HttpRestConfig; +import com.dtstack.chunjun.converter.AbstractRowConverter; + +import com.csvreader.CsvReader; +import org.apache.commons.collections.CollectionUtils; + +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.util.HashMap; +import java.util.Map; + +public class CsvResponseParse extends ResponseParse { + private CsvReader csvReader; + private Reader reader; + private String responseValue; + private HttpRequestParam requestParam; + + public CsvResponseParse(HttpRestConfig config, AbstractRowConverter converter) { + super(config, converter); + if (CollectionUtils.isEmpty(columns)) { + throw new RuntimeException("please configure column when decode is csv"); + } + } + + @Override + public boolean hasNext() throws IOException { + return csvReader.readRecord(); + } + + @Override + public ResponseValue next() throws Exception { + String[] data = csvReader.getValues(); + HashMap stringObjectHashMap = new HashMap<>(columns.size()); + for (int i = 0; i < columns.size(); i++) { + if (columns.get(i).getValue() != null) { + stringObjectHashMap.put(columns.get(i).getName(), columns.get(i).getValue()); + } else { + stringObjectHashMap.put(columns.get(i).getName(), data[i]); + } + } + + return new ResponseValue( + converter.toInternal(stringObjectHashMap), requestParam, responseValue); + } + + @Override + public void parse(String responseValue, int responseStatus, HttpRequestParam requestParam) { + this.responseValue = responseValue; + this.requestParam = requestParam; + this.reader = new StringReader(responseValue); + this.csvReader = new CsvReader(reader); + csvReader.setDelimiter(config.getCsvDelimiter().charAt(0)); + + Map csvConfig = config.getCsvConfig(); + // 是否跳过空行 + csvReader.setSkipEmptyRecords((Boolean) csvConfig.getOrDefault("skipEmptyRecords", true)); + // 是否使用csv转义字符 + csvReader.setUseTextQualifier((Boolean) csvConfig.getOrDefault("useTextQualifier", true)); + csvReader.setTrimWhitespace((Boolean) csvConfig.getOrDefault("trimWhitespace", false)); + // 单列长度是否限制100000字符 + csvReader.setSafetySwitch((Boolean) csvConfig.getOrDefault("safetySwitch", false)); + } +} diff --git a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/DefaultRestHandler.java b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/DefaultRestHandler.java index 2c31477a93..2ad2bca056 100644 --- a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/DefaultRestHandler.java +++ b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/DefaultRestHandler.java @@ -163,12 +163,6 @@ public HttpRequestParam buildRequestParam( return requestParam; } - @Override - public ResponseValue buildResponseValue( - String decode, String responseValue, String fields, HttpRequestParam requestParam) { - return new ResponseValue(responseValue, requestParam, responseValue); - } - /** * 根据指定的key 构建一个新的response * diff --git a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/HttpClient.java b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/HttpClient.java index 821d5a0fc1..cb6fce77ae 100644 --- a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/HttpClient.java +++ b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/HttpClient.java @@ -21,6 +21,7 @@ import com.dtstack.chunjun.connector.http.common.HttpRestConfig; import com.dtstack.chunjun.connector.http.common.HttpUtil; import com.dtstack.chunjun.connector.http.common.MetaParam; +import com.dtstack.chunjun.converter.AbstractRowConverter; import com.dtstack.chunjun.util.ExceptionUtil; import com.dtstack.chunjun.util.GsonUtil; @@ -41,6 +42,10 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import static com.dtstack.chunjun.connector.http.common.ConstantValue.CSV_DECODE; +import static com.dtstack.chunjun.connector.http.common.ConstantValue.TEXT_DECODE; +import static com.dtstack.chunjun.connector.http.common.ConstantValue.XML_DECODE; + /** * httpClient * @@ -60,6 +65,8 @@ public class HttpClient { private final RestHandler restHandler; + protected final ResponseParse responseParse; + private int requestRetryTime; /** origin body */ @@ -86,11 +93,14 @@ public class HttpClient { private boolean running; + protected long requestNumber; + public HttpClient( HttpRestConfig httpRestConfig, List originalBodyList, List originalParamList, - List originalHeaderList) { + List originalHeaderList, + AbstractRowConverter converter) { this.restConfig = httpRestConfig; this.originalHeaderList = originalHeaderList; this.originalBodyList = originalBodyList; @@ -102,14 +112,16 @@ public HttpClient( this.queue = new LinkedBlockingQueue<>(); this.scheduledExecutorService = new ScheduledThreadPoolExecutor(1, r -> new Thread(r, THREAD_NAME)); - this.httpClient = HttpUtil.getHttpsClient(); + this.httpClient = HttpUtil.getHttpsClient((int) restConfig.getTimeOut()); this.restHandler = new DefaultRestHandler(); + this.responseParse = getResponseParse(converter); this.prevResponse = ""; this.first = true; this.currentParam = new HttpRequestParam(); this.reachEnd = false; this.requestRetryTime = 2; + this.requestNumber = 1; } public void start() { @@ -172,6 +184,7 @@ public void execute() { doExecute(ConstantValue.REQUEST_RETRY_TIME); first = false; requestRetryTime = 3; + requestNumber++; } public void doExecute(int retryTime) { @@ -192,6 +205,7 @@ public void doExecute(int retryTime) { // 执行请求 String responseValue = null; + int responseStatus; try { HttpUriRequest request = @@ -211,6 +225,7 @@ public void doExecute(int retryTime) { } responseValue = EntityUtils.toString(httpResponse.getEntity()); + responseStatus = httpResponse.getStatusLine().getStatusCode(); } catch (Throwable e) { // 只要本次请求中出现了异常 都会进行重试，如果重试次数达到了就真正结束任务 LOG.warn( @@ -244,27 +259,28 @@ public void doExecute(int retryTime) { case ConstantValue.STRATEGY_STOP: reachEnd = true; running = false; + // stop 此次请求数据有问题 任务直接异常结束 + processData(new ResponseValue(0, null, strategy.toString(), null, null)); break; default: break; } } - ResponseValue value = - restHandler.buildResponseValue( - restConfig.getDecode(), - responseValue, - restConfig.getFields(), - HttpRequestParam.copy(currentParam)); + responseParse.parse(responseValue, responseStatus, HttpRequestParam.copy(currentParam)); + while (responseParse.hasNext()) { + processData(responseParse.next()); + } + + if (-1 != restConfig.getCycles() && requestNumber >= restConfig.getCycles()) { + reachEnd = true; + running = false; + } + if (reachEnd) { // 如果结束了 需要告诉format 结束了 - if (value.isNormal()) { - value.setStatus(0); - // 触发的策略信息返回上游 - value.setErrorMsg(strategy.toString()); - } + processData(new ResponseValue(2, null, null, null, null)); } - processData(value); prevParam = currentParam; prevResponse = responseValue; @@ -321,6 +337,19 @@ public void close() { } } + protected ResponseParse getResponseParse(AbstractRowConverter converter) { + switch (restConfig.getDecode()) { + case CSV_DECODE: + return new CsvResponseParse(restConfig, converter); + case XML_DECODE: + return new XmlResponseParse(restConfig, converter); + case TEXT_DECODE: + return new TextResponseParse(restConfig, converter); + default: + return new JsonResponseParse(restConfig, converter); + } + } + @Override public String toString() { return "HttpClient{" diff --git a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/JsonResponseParse.java b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/JsonResponseParse.java new file mode 100644 index 0000000000..67768815a1 --- /dev/null +++ b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/JsonResponseParse.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.connector.http.client; + +import com.dtstack.chunjun.conf.FieldConf; +import com.dtstack.chunjun.connector.http.common.HttpRestConfig; +import com.dtstack.chunjun.connector.http.util.JsonPathUtil; +import com.dtstack.chunjun.constants.ConstantValue; +import com.dtstack.chunjun.converter.AbstractRowConverter; +import com.dtstack.chunjun.util.GsonUtil; +import com.dtstack.chunjun.util.MapUtil; + +import com.google.common.collect.Lists; +import com.google.gson.Gson; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class JsonResponseParse extends ResponseParse { + private String responseValue; + private HttpRequestParam requestParam; + private final Gson gson; + private final List fields; + private Iterator> iterator; + + public JsonResponseParse(HttpRestConfig config, AbstractRowConverter converter) { + super(config, converter); + this.gson = GsonUtil.setTypeAdapter(new Gson()); + if (StringUtils.isNotBlank(config.getFields())) { + fields = + Arrays.stream(config.getFields().split(",")) + .map( + i -> { + FieldConf fieldConf = new FieldConf(); + fieldConf.setName(i); + return fieldConf; + }) + .collect(Collectors.toList()); + } else { + fields = null; + } + } + + @Override + public boolean hasNext() throws IOException { + return iterator.hasNext(); + } + + @Override + public ResponseValue next() throws Exception { + Map next = iterator.next(); + + if (CollectionUtils.isEmpty(columns)) { + if (CollectionUtils.isNotEmpty(fields)) { + LinkedHashMap map = + buildResponseByKey(next, fields, ConstantValue.POINT_SYMBOL); + HashMap data = new HashMap<>(); + // 需要拆分key + ((Map) map) + .forEach( + (k, v) -> { + MapUtil.buildMap(k, ConstantValue.POINT_SYMBOL, v, data); + }); + return new ResponseValue(converter.toInternal(data), requestParam, responseValue); + } else { + return new ResponseValue(converter.toInternal(next), requestParam, responseValue); + } + + } else { + LinkedHashMap data = + buildResponseByKey(next, columns, ConstantValue.POINT_SYMBOL); + return new ResponseValue(converter.toInternal(data), requestParam, responseValue); + } + } + + @Override + public void parse(String responseValue, int responseStatus, HttpRequestParam requestParam) { + this.responseValue = responseValue; + this.requestParam = requestParam; + + Map map = gson.fromJson(responseValue, GsonUtil.gsonMapTypeToken); + if (StringUtils.isNotBlank(config.getDataSubject())) { + Object valueByKey = + MapUtil.getValueByKey( + map, + JsonPathUtil.parseJsonPath(config.getDataSubject()), + ConstantValue.POINT_SYMBOL); + if (valueByKey instanceof List) { + this.iterator = ((List) valueByKey).iterator(); + } else { + throw new RuntimeException(config.getDataSubject() + " in response is not array"); + } + } else { + this.iterator = Lists.newArrayList(map).iterator(); + } + } +} diff --git a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/ResponseParse.java b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/ResponseParse.java new file mode 100644 index 0000000000..b3d4cfc5b1 --- /dev/null +++ b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/ResponseParse.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.connector.http.client; + +import com.dtstack.chunjun.conf.FieldConf; +import com.dtstack.chunjun.connector.http.common.HttpRestConfig; +import com.dtstack.chunjun.converter.AbstractRowConverter; +import com.dtstack.chunjun.util.GsonUtil; +import com.dtstack.chunjun.util.MapUtil; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public abstract class ResponseParse { + protected final HttpRestConfig config; + protected final List columns; + protected final AbstractRowConverter converter; + + public ResponseParse(HttpRestConfig config, AbstractRowConverter converter) { + this.config = config; + this.columns = config.getColumn(); + this.converter = converter; + } + + public abstract boolean hasNext() throws IOException; + + public abstract ResponseValue next() throws Exception; + + public abstract void parse( + String responseValue, int responseStatus, HttpRequestParam requestParam); + + /** + * 根据指定的key 构建一个新的response + * + * @param map 返回值 + * @param columns 指定字段 + */ + protected LinkedHashMap buildResponseByKey( + Map map, List columns, String nested) { + LinkedHashMap filedValue = new LinkedHashMap<>(columns.size() << 2); + for (FieldConf key : columns) { + if (null != key.getValue()) { + filedValue.put(key.getName(), key.getValue()); + } else { + Object value = MapUtil.getValueByKey(map, key.getName(), nested); + filedValue.put(key.getName(), value); + } + } + return filedValue; + } + + protected LinkedHashMap buildResponseByKey( + Map map, List columns, boolean useNullReplaceNotExists) { + LinkedHashMap filedValue = new LinkedHashMap<>(columns.size() << 2); + for (FieldConf key : columns) { + if (null != key.getValue()) { + filedValue.put(key.getName(), key.getValue()); + } else { + if (!map.containsKey(key.getName())) { + if (useNullReplaceNotExists) { + filedValue.put(key.getName(), null); + } else { + throw new RuntimeException( + "not exists column " + + key.getName() + + " in map: " + + GsonUtil.GSON.toJson(map)); + } + } else { + Object o = map.get(key.getName()); + filedValue.put(key.getName(), o); + } + } + } + return filedValue; + } +} diff --git a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/ResponseValue.java b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/ResponseValue.java index b589cadf3a..073cd5a278 100644 --- a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/ResponseValue.java +++ b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/ResponseValue.java @@ -18,6 +18,8 @@ package com.dtstack.chunjun.connector.http.client; +import org.apache.flink.table.data.RowData; + /** * 返回值 * @@ -28,7 +30,7 @@ public class ResponseValue { /** 本次请求状态 -1 不正常，代表出现了异常 0 代表结束任务 strategy 出现了stop 1 代表任务正常 */ private int status; /** 返回值 */ - private String data; + private RowData data; /** 如果是异常数据 这个是异常数据 */ private String errorMsg; /** 请求参数 */ @@ -39,7 +41,7 @@ public class ResponseValue { public ResponseValue( int status, - String data, + RowData data, String errorMsg, HttpRequestParam requestParam, String originResponseValue) { @@ -50,7 +52,7 @@ public ResponseValue( this.originResponseValue = originResponseValue; } - public ResponseValue(String data, HttpRequestParam requestParam, String originResponseValue) { + public ResponseValue(RowData data, HttpRequestParam requestParam, String originResponseValue) { this(1, data, null, requestParam, originResponseValue); } @@ -58,7 +60,7 @@ public boolean isNormal() { return status != -1; } - public String getData() { + public RowData getData() { return data; } diff --git a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/RestHandler.java b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/RestHandler.java index 3e49ac7946..fa231377d0 100644 --- a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/RestHandler.java +++ b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/RestHandler.java @@ -68,15 +68,4 @@ HttpRequestParam buildRequestParam( Map prevResponseValue, HttpRestConfig restConfig, boolean first); - - /** - * 根据返回的response 构建出ResponseValue * json格式 会指定字段解析 - * - * @param decode 解析格式 json还是text - * @param responseValue 返回值 - * @param fields 解析字段 - * @return 返回值 - */ - ResponseValue buildResponseValue( - String decode, String responseValue, String fields, HttpRequestParam requestParam); } diff --git a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/TextResponseParse.java b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/TextResponseParse.java new file mode 100644 index 0000000000..2aca7a9920 --- /dev/null +++ b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/TextResponseParse.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.connector.http.client; + +import com.dtstack.chunjun.connector.http.common.HttpRestConfig; +import com.dtstack.chunjun.converter.AbstractRowConverter; +import com.dtstack.chunjun.element.ColumnRowData; +import com.dtstack.chunjun.element.column.StringColumn; + +import com.google.common.collect.Lists; + +import java.io.IOException; +import java.util.Iterator; + +public class TextResponseParse extends ResponseParse { + private Iterator iterator; + private HttpRequestParam requestParam; + private String responseValue; + + public TextResponseParse(HttpRestConfig config, AbstractRowConverter converter) { + super(config, converter); + } + + @Override + public boolean hasNext() throws IOException { + return iterator.hasNext(); + } + + @Override + public ResponseValue next() throws Exception { + ColumnRowData columnRowData = new ColumnRowData(1); + columnRowData.addField(new StringColumn(iterator.next())); + return new ResponseValue(columnRowData, requestParam, responseValue); + } + + @Override + public void parse(String responseValue, int responseStatus, HttpRequestParam requestParam) { + this.responseValue = responseValue; + this.requestParam = requestParam; + this.iterator = Lists.newArrayList(responseValue).iterator(); + } +} diff --git a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/XmlResponseParse.java b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/XmlResponseParse.java new file mode 100644 index 0000000000..fd5e6e88e1 --- /dev/null +++ b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/client/XmlResponseParse.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.connector.http.client; + +import com.dtstack.chunjun.connector.http.common.HttpRestConfig; +import com.dtstack.chunjun.connector.http.util.JsonPathUtil; +import com.dtstack.chunjun.connector.http.util.XmlUtil; +import com.dtstack.chunjun.constants.ConstantValue; +import com.dtstack.chunjun.converter.AbstractRowConverter; +import com.dtstack.chunjun.util.MapUtil; + +import com.google.common.collect.Lists; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class XmlResponseParse extends ResponseParse { + private String responseValue; + private HttpRequestParam requestParam; + private Iterator> iterator; + + public XmlResponseParse(HttpRestConfig config, AbstractRowConverter converter) { + super(config, converter); + if (CollectionUtils.isEmpty(columns)) { + throw new RuntimeException("please configure column when decode is csv"); + } + } + + @Override + public boolean hasNext() throws IOException { + return iterator.hasNext(); + } + + @Override + public ResponseValue next() throws Exception { + Map next = iterator.next(); + if (StringUtils.isBlank(config.getDataSubject()) + && next.containsKey(columns.get(0).getName())) { + // rootKey + String key = columns.get(0).getName(); + HashMap data = new HashMap<>(); + data.put(key, next.get(key)); + return new ResponseValue(converter.toInternal(data), requestParam, responseValue); + } + LinkedHashMap data = + buildResponseByKey(next, columns, ConstantValue.POINT_SYMBOL); + return new ResponseValue(converter.toInternal(data), requestParam, responseValue); + } + + @Override + public void parse(String responseValue, int responseStatus, HttpRequestParam requestParam) { + this.responseValue = responseValue; + this.requestParam = requestParam; + + Map xmlData = new HashMap<>(); + XmlUtil.xmlParse(responseValue, xmlData); + if (StringUtils.isBlank(config.getDataSubject())) { + this.iterator = Lists.newArrayList(xmlData).iterator(); + } else { + Object valueByKey = + MapUtil.getValueByKey( + xmlData, + JsonPathUtil.parseJsonPath(config.getDataSubject()), + ConstantValue.POINT_SYMBOL); + if (valueByKey instanceof List) { + this.iterator = ((List) valueByKey).iterator(); + } else if (valueByKey instanceof Map) { + Map data = (Map) valueByKey; + this.iterator = Lists.newArrayList(data).iterator(); + } else { + throw new RuntimeException( + config.getDataSubject() + " in response is not array or map"); + } + } + } +} diff --git a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/common/ConstantValue.java b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/common/ConstantValue.java index ee5ad97979..c0d22b7979 100644 --- a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/common/ConstantValue.java +++ b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/common/ConstantValue.java @@ -35,6 +35,10 @@ public class ConstantValue { public static final String SYSTEM_FUNCTION_CURRENT_TIME = "currentTime"; public static final String SYSTEM_FUNCTION_INTERVAL_TIME = "intervalTime"; + public static final String CSV_DECODE = "csv"; + public static final String XML_DECODE = "xml"; + public static final String TEXT_DECODE = "text"; + public static final String DEFAULT_DECODE = "json"; public static String PREFIX = "${"; diff --git a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/common/HttpRestConfig.java b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/common/HttpRestConfig.java index 4cec16ccf4..cc76833d0c 100644 --- a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/common/HttpRestConfig.java +++ b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/common/HttpRestConfig.java @@ -21,7 +21,9 @@ import com.dtstack.chunjun.connector.http.client.Strategy; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * HttpRestConfig @@ -43,6 +45,13 @@ public class HttpRestConfig extends ChunJunCommonConf { private String fieldDelimiter = com.dtstack.chunjun.constants.ConstantValue.POINT_SYMBOL; + /** 数据主体，代表json或者xml返回数据里，数据主体字段对应的数据一定是一个数组，里面的数据需要拆分 */ + private String dataSubject; + + private String csvDelimiter = com.dtstack.chunjun.constants.ConstantValue.COMMA_SYMBOL; + + private Map csvConfig = new HashMap<>(); + /** response text/json */ private String decode = "text"; @@ -53,7 +62,11 @@ public class HttpRestConfig extends ChunJunCommonConf { private String fieldTypes; /** 请求的间隔时间 单位毫秒 */ - private Long intervalTime; + private Long intervalTime = 3000L; + + // allow request num cycles 为-1时，除非异常或者strategy生效导致任务结束，否则任务会一直循环请求，如果 大于 + // 0，则代表循环请求的次数，如配置为3，则会发送三次http请求 + private long cycles = -1; /** 请求的header头 */ private List header = new ArrayList<>(2); @@ -67,6 +80,9 @@ public class HttpRestConfig extends ChunJunCommonConf { /** 返回结果的处理策略 */ protected List strategy = new ArrayList<>(2); + /** 请求的超时时间 单位毫秒 */ + private long timeOut = 10000; + public String getFieldTypes() { return fieldTypes; } @@ -167,6 +183,46 @@ public void setFieldDelimiter(String fieldDelimiter) { this.fieldDelimiter = fieldDelimiter; } + public String getDataSubject() { + return dataSubject; + } + + public void setDataSubject(String dataSubject) { + this.dataSubject = dataSubject; + } + + public String getCsvDelimiter() { + return csvDelimiter; + } + + public void setCsvDelimiter(String csvDelimiter) { + this.csvDelimiter = csvDelimiter; + } + + public Map getCsvConfig() { + return csvConfig; + } + + public void setCsvConfig(Map csvConfig) { + this.csvConfig = csvConfig; + } + + public long getCycles() { + return cycles; + } + + public void setCycles(long cycles) { + this.cycles = cycles; + } + + public long getTimeOut() { + return timeOut; + } + + public void setTimeOut(long timeOut) { + this.timeOut = timeOut; + } + @Override public String toString() { return "HttpRestConfig{" @@ -179,16 +235,30 @@ public String toString() { + ", requestMode='" + requestMode + '\'' + + ", fieldDelimiter='" + + fieldDelimiter + + '\'' + + ", dataSubject='" + + dataSubject + + '\'' + + ", csvDelimiter='" + + csvDelimiter + + '\'' + + ", csvConfig=" + + csvConfig + ", decode='" + decode + '\'' + ", fields='" + fields + '\'' + + ", fieldTypes='" + + fieldTypes + + '\'' + ", intervalTime=" + intervalTime - + ", fieldDelimiter=" - + fieldDelimiter + + ", cycles=" + + cycles + ", header=" + header + ", param=" @@ -197,6 +267,8 @@ public String toString() { + body + ", strategy=" + strategy + + ", timeOut=" + + timeOut + '}'; } } diff --git a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/common/HttpUtil.java b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/common/HttpUtil.java index 785235dc33..00a99f5b40 100644 --- a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/common/HttpUtil.java +++ b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/common/HttpUtil.java @@ -58,10 +58,10 @@ public class HttpUtil { public static CloseableHttpClient getHttpClient() { - return getBaseBuilder().build(); + return getBaseBuilder(TIME_OUT).build(); } - public static CloseableHttpClient getHttpsClient() { + public static CloseableHttpClient getHttpsClient(int timeOut) { // 设置Http连接池 SSLContext sslContext; @@ -74,13 +74,16 @@ public static CloseableHttpClient getHttpsClient() { LOG.warn(ExceptionUtil.getErrorMessage(e)); throw new RuntimeException(e); } - return getBaseBuilder() + return getBaseBuilder(timeOut) .setSSLContext(sslContext) .setSSLHostnameVerifier(new NoopHostnameVerifier()) .build(); } - public static HttpClientBuilder getBaseBuilder() { + public static HttpClientBuilder getBaseBuilder(int timeOut) { + if (timeOut <= 0) { + timeOut = TIME_OUT; + } // 设置自定义的重试策略 ServiceUnavailableRetryStrategyImpl strategy = new ServiceUnavailableRetryStrategyImpl.Builder() @@ -93,9 +96,9 @@ public static HttpClientBuilder getBaseBuilder() { // 设置超时时间 RequestConfig requestConfig = RequestConfig.custom() - .setConnectTimeout(TIME_OUT) - .setConnectionRequestTimeout(TIME_OUT) - .setSocketTimeout(TIME_OUT) + .setConnectTimeout(timeOut) + .setConnectionRequestTimeout(timeOut) + .setSocketTimeout(timeOut) .build(); // 设置Http连接池 PoolingHttpClientConnectionManager pcm = new PoolingHttpClientConnectionManager(); diff --git a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/converter/HttpColumnConverter.java b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/converter/HttpColumnConverter.java index ed1e977ad3..729fc668dd 100644 --- a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/converter/HttpColumnConverter.java +++ b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/converter/HttpColumnConverter.java @@ -18,8 +18,7 @@ package com.dtstack.chunjun.connector.http.converter; -import com.dtstack.chunjun.connector.http.client.DefaultRestHandler; -import com.dtstack.chunjun.connector.http.common.ConstantValue; +import com.dtstack.chunjun.conf.FieldConf; import com.dtstack.chunjun.connector.http.common.HttpRestConfig; import com.dtstack.chunjun.converter.AbstractRowConverter; import com.dtstack.chunjun.converter.IDeserializationConverter; @@ -32,8 +31,6 @@ import com.dtstack.chunjun.element.column.StringColumn; import com.dtstack.chunjun.element.column.TimestampColumn; import com.dtstack.chunjun.util.DateUtil; -import com.dtstack.chunjun.util.GsonUtil; -import com.dtstack.chunjun.util.MapUtil; import org.apache.flink.table.data.RowData; @@ -41,6 +38,7 @@ import java.math.BigDecimal; import java.util.ArrayList; +import java.util.List; import java.util.Locale; import java.util.Map; @@ -50,7 +48,7 @@ * @description */ public class HttpColumnConverter - extends AbstractRowConverter, String> { + extends AbstractRowConverter, Object, Map, String> { /** restapi Conf */ private HttpRestConfig httpRestConfig; @@ -77,34 +75,27 @@ protected ISerializationConverter> wrapIntoNullableExternalC } @Override - public RowData toInternal(String input) throws Exception { + public RowData toInternal(Map input) throws Exception { ColumnRowData row; - if (httpRestConfig.getDecode().equals(ConstantValue.DEFAULT_DECODE)) { - Map result = - DefaultRestHandler.gson.fromJson(input, GsonUtil.gsonMapTypeToken); - if (toInternalConverters != null && toInternalConverters.size() > 0) { - // 同步任务配置了field参数(对应的类型转换都是string) 需要对每个字段进行类型转换 - row = new ColumnRowData(toInternalConverters.size()); - String fields = httpRestConfig.getFields(); - String[] split = fields.split(","); - - for (int i = 0; i < split.length; i++) { - Object value = - MapUtil.getValueByKey( - result, split[i], httpRestConfig.getFieldDelimiter()); - row.addField( - (AbstractBaseColumn) toInternalConverters.get(i).deserialize(value)); - } - } else { - // 直接作为mapColumn - row = new ColumnRowData(1); - row.addField(new MapColumn(result)); - } + if (toInternalConverters != null && toInternalConverters.size() > 0) { + List fieldConfList = commonConf.getColumn(); + // 同步任务配置了field参数(对应的类型转换都是string) 需要对每个字段进行类型转换 + row = new ColumnRowData(toInternalConverters.size()); + for (int i = 0; i < toInternalConverters.size(); i++) { + String name = httpRestConfig.getColumn().get(i).getName(); + AbstractBaseColumn baseColumn = + (AbstractBaseColumn) + toInternalConverters.get(i).deserialize(input.get(name)); + + row.addField(assembleFieldProps(fieldConfList.get(i), baseColumn)); + } } else { + // 实时直接作为mapColumn row = new ColumnRowData(1); - row.addField(new StringColumn(input)); + row.addField(new MapColumn(input)); } + return row; } diff --git a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/inputformat/HttpInputFormat.java b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/inputformat/HttpInputFormat.java index daf3e96e2f..b0ace723b1 100644 --- a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/inputformat/HttpInputFormat.java +++ b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/inputformat/HttpInputFormat.java @@ -69,7 +69,8 @@ public void openInputFormat() throws IOException { @Override @SuppressWarnings("unchecked") protected void openInternal(InputSplit inputSplit) { - myHttpClient = new HttpClient(httpRestConfig, metaBodys, metaParams, metaHeaders); + myHttpClient = + new HttpClient(httpRestConfig, metaBodys, metaParams, metaHeaders, rowConverter); if (state != null) { myHttpClient.initPosition(state.getRequestParam(), state.getOriginResponseValue()); } @@ -96,14 +97,20 @@ protected RowData nextRecordInternal(RowData rowData) throws ReadRecordException + value.getOriginResponseValue() + " job end"); } + // finished 任务正常结束 + if (value.getStatus() == 2) { + reachEnd = true; + return null; + } + // todo 离线任务后期需要加上一个finished策略 这样就是代表任务正常结束 而不是异常stop state = new ResponseValue( - "", + null, HttpRequestParam.copy(value.getRequestParam()), value.getOriginResponseValue()); try { - return rowConverter.toInternal(value.getData()); + return value.getData(); } catch (Exception e) { throw new ReadRecordException(e.getMessage(), e); } diff --git a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/util/JsonPathUtil.java b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/util/JsonPathUtil.java new file mode 100644 index 0000000000..e680364629 --- /dev/null +++ b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/util/JsonPathUtil.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.connector.http.util; + +import org.apache.commons.lang3.StringUtils; + +public class JsonPathUtil { + public static String parseJsonPath(String path) { + // + if (path.startsWith("$.")) { + return path.substring(2); + } else if (path.startsWith("${") && path.endsWith("}")) { + return path.substring(2, path.length() - 1); + } + throw new IllegalArgumentException("just support parse ${xxx.xxx} or $.xxx.xx format"); + } + + public static boolean isJsonPath(String path) { + if (StringUtils.isBlank(path)) { + throw new IllegalArgumentException("path is not empty"); + } + return path.startsWith("$.") || (path.startsWith("${") && path.endsWith("}")); + } +} diff --git a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/util/XmlUtil.java b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/util/XmlUtil.java new file mode 100644 index 0000000000..d62d31d22a --- /dev/null +++ b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/util/XmlUtil.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.connector.http.util; + +import org.apache.commons.lang3.tuple.Pair; +import org.dom4j.Attribute; +import org.dom4j.Document; +import org.dom4j.DocumentException; +import org.dom4j.DocumentHelper; +import org.dom4j.Element; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class XmlUtil { + + public static Map xmlParse(String xml, Map outmap) { + Document document = null; + try { + document = DocumentHelper.parseText(xml); + } catch (DocumentException e) { + throw new RuntimeException( + "parse xml failed,the response values is not xml format and value is ->" + xml); + } + // 3.获取根节点 + Element rootElement = document.getRootElement(); + + return elementTomap(rootElement, outmap); + } + + @SuppressWarnings("unchecked") + public static Map elementTomap(Element outele, Map outmap) { + List list = outele.elements(); + int size = list.size(); + if (size == 0) { + outmap.put(outele.getName(), outele.getTextTrim()); + } else { + Map innermap = new HashMap<>(); + for (Element ele1 : list) { + String eleName = ele1.getName(); + Object obj = innermap.get(eleName); + if (obj == null) { + elementTomap(ele1, innermap); + } else { + if (obj instanceof Map) { + List> list1 = new ArrayList>(); + list1.add((Map) innermap.remove(eleName)); + elementTomap(ele1, innermap); + list1.add((Map) innermap.remove(eleName)); + innermap.put(eleName, list1); + } else if (obj instanceof String) { + List list1 = new ArrayList(); + list1.add((String) innermap.remove(eleName)); + elementTomap(ele1, innermap); + list1.add((String) innermap.remove(eleName)); + innermap.put(eleName, list1); + } else { + elementTomap(ele1, innermap); + if (innermap.get(eleName) instanceof Map) { + Map listValue = + (Map) innermap.get(eleName); + ((List>) obj).add(listValue); + innermap.put(eleName, obj); + } else if (innermap.get(eleName) instanceof String) { + String listValue = (String) innermap.get(eleName); + ((List) obj).add(listValue); + innermap.put(eleName, obj); + } + } + } + } + outmap.put(outele.getName(), innermap); + } + return outmap; + } + + public static LinkedHashMap>> hbaseXmlParse(String xml) { + Document document = null; + try { + document = DocumentHelper.parseText(xml); + } catch (DocumentException e) { + throw new RuntimeException( + "parse xml failed,the response values is not xml format and value is ->" + xml); + } + // 3.获取根节点 + Element rootElement = document.getRootElement(); + + return hbaseElementTomap(rootElement); + } + + // example + // + // + // MTM= + // a3lvdG9t + // + // + // MTY= + // + // + + public static LinkedHashMap>> hbaseElementTomap( + Element outele) { + + LinkedHashMap>> data = new LinkedHashMap<>(); + List list = outele.elements(); + for (Element ele1 : list) { + String eleName = ele1.getName(); + if (!eleName.equalsIgnoreCase("Row")) { + continue; + } + Attribute key = ele1.attribute("key"); + if (key == null) { + throw new RuntimeException( + "parse hbase xml data error, row element not has attribute key"); + } + // cell + ArrayList> columns = new ArrayList<>(); + // add rowkey + columns.add(Pair.of("rowkey", key.getValue())); + for (Element element : ele1.elements()) { + if (element.getName().equalsIgnoreCase("Cell")) { + Attribute column = element.attribute("column"); + if (column == null) { + throw new RuntimeException( + "parse hbase xml data error, cell element has not attribute column"); + } + columns.add(Pair.of(column.getValue(), element.getText())); + } + } + + data.put(key.getValue(), columns); + } + + return data; + } +} From 986a026703655710e3a90c5f6ca5575a9cacd54c Mon Sep 17 00:00:00 2001 From: taoyameng <54356452+taoyameng@users.noreply.github.com> Date: Mon, 13 Feb 2023 16:31:24 +0800 Subject: [PATCH 30/41] [hotfix-#1497][chunjun-connector-http]Solve the problem that http-x plug-in cannot be used in sql mode (#1500) Co-authored-by: 443321070@qq.com (cherry picked from commit d65793b4286a32986587e4abb3effd4089e6045a) --- .../connector/http/converter/HttpRowConverter.java | 9 +++------ .../http/table/HttpDynamicTableFactory.java | 4 ++++ .../chunjun/connector/http/table/HttpOptions.java | 12 ++++++++++++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/converter/HttpRowConverter.java b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/converter/HttpRowConverter.java index 8994c61b78..9aa8331813 100644 --- a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/converter/HttpRowConverter.java +++ b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/converter/HttpRowConverter.java @@ -18,12 +18,10 @@ package com.dtstack.chunjun.connector.http.converter; -import com.dtstack.chunjun.connector.http.client.DefaultRestHandler; import com.dtstack.chunjun.connector.http.common.HttpRestConfig; import com.dtstack.chunjun.converter.AbstractRowConverter; import com.dtstack.chunjun.converter.IDeserializationConverter; import com.dtstack.chunjun.converter.ISerializationConverter; -import com.dtstack.chunjun.util.GsonUtil; import com.dtstack.chunjun.util.MapUtil; import org.apache.flink.table.data.DecimalData; @@ -45,7 +43,8 @@ import java.util.Map; /** Base class for all converters that convert between restapi body and Flink internal object. */ -public class HttpRowConverter extends AbstractRowConverter { +public class HttpRowConverter + extends AbstractRowConverter, RowData, RowData, LogicalType> { private HttpRestConfig httpRestConfig; @@ -69,9 +68,7 @@ public HttpRowConverter(RowType rowType) { } @Override - public RowData toInternal(String input) throws Exception { - Map result = - DefaultRestHandler.gson.fromJson(input, GsonUtil.gsonMapTypeToken); + public RowData toInternal(Map result) throws Exception { GenericRowData genericRowData = new GenericRowData(rowType.getFieldCount()); List columns = rowType.getFieldNames(); for (int pos = 0; pos < columns.size(); pos++) { diff --git a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/table/HttpDynamicTableFactory.java b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/table/HttpDynamicTableFactory.java index a8752280f2..7e17812605 100644 --- a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/table/HttpDynamicTableFactory.java +++ b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/table/HttpDynamicTableFactory.java @@ -72,6 +72,8 @@ public Set> optionalOptions() { options.add(HttpOptions.INTERVALTIME); options.add(HttpOptions.COLUMN); options.add(HttpOptions.DELAY); + options.add(HttpOptions.DATA_SUBJECT); + options.add(HttpOptions.CYCLES); return options; } @@ -142,6 +144,8 @@ private HttpRestConfig getRestapiConf(ReadableConfig config) { httpRestConfig.setUrl(config.get(HttpOptions.URL)); httpRestConfig.setDecode(config.get(HttpOptions.DECODE)); httpRestConfig.setRequestMode(config.get(HttpOptions.METHOD)); + httpRestConfig.setDataSubject(config.get(HttpOptions.DATA_SUBJECT)); + httpRestConfig.setCycles(config.get(HttpOptions.CYCLES)); httpRestConfig.setParam( gson.fromJson( config.get(HttpOptions.PARAMS), diff --git a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/table/HttpOptions.java b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/table/HttpOptions.java index 81ea37ba0f..0bfaf10eb5 100644 --- a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/table/HttpOptions.java +++ b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/table/HttpOptions.java @@ -64,4 +64,16 @@ public class HttpOptions { public static final ConfigOption DELAY = ConfigOptions.key("delay").intType().defaultValue(30).withDescription("request delay"); + + public static final ConfigOption DATA_SUBJECT = + ConfigOptions.key("dataSubject") + .stringType() + .defaultValue("${data}") + .withDescription("response data subject"); + + public static final ConfigOption CYCLES = + ConfigOptions.key("cycles") + .longType() + .defaultValue(1L) + .withDescription("request cycle"); } From d5dae2ffb9923fb2e5dafec035eca5ea21d84d74 Mon Sep 17 00:00:00 2001 From: Paddy Date: Thu, 16 Feb 2023 11:54:56 +0800 Subject: [PATCH 31/41] [feat-#1498][main] support run sql job with batch mode when set runMode=batch (#1504) (cherry picked from commit 83c10dec3d8d57ddfc88e2d9689c7af52ea94e62) --- .../greenplum/dialect/GreenplumDialect.java | 5 ----- .../greenplum/sink/GreenplumOutputFormat.java | 4 ---- .../greenplum/sink/GreenplumSinkFactory.java | 5 ----- .../src/main/java/com/dtstack/chunjun/Main.java | 3 +++ .../java/com/dtstack/chunjun/options/Options.java | 13 +++++++++++++ 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/dialect/GreenplumDialect.java b/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/dialect/GreenplumDialect.java index 62045c9e0b..56b2e64ca2 100644 --- a/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/dialect/GreenplumDialect.java +++ b/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/dialect/GreenplumDialect.java @@ -24,11 +24,6 @@ import java.util.Optional; -/** - * company www.dtstack.com - * - * @author jier - */ public class GreenplumDialect extends PostgresqlDialect { private static final String DIALECT_NAME = "Greenplum"; diff --git a/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/sink/GreenplumOutputFormat.java b/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/sink/GreenplumOutputFormat.java index 9c19d5cdfc..8769e69c20 100644 --- a/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/sink/GreenplumOutputFormat.java +++ b/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/sink/GreenplumOutputFormat.java @@ -41,10 +41,6 @@ import java.sql.Connection; import java.sql.SQLException; -/** - * @program: flinkx - * @author: jier - */ public class GreenplumOutputFormat extends JdbcOutputFormat { // pg 字符串里含有\u0000 会报错 ERROR: invalid byte sequence for encoding "UTF8": 0x00 diff --git a/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/sink/GreenplumSinkFactory.java b/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/sink/GreenplumSinkFactory.java index fa38a5d998..a27638644f 100644 --- a/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/sink/GreenplumSinkFactory.java +++ b/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/sink/GreenplumSinkFactory.java @@ -28,11 +28,6 @@ import static com.dtstack.chunjun.connector.greenplum.sink.GreenplumOutputFormat.INSERT_SQL_MODE_TYPE; -/** - * company www.dtstack.com - * - * @author jier - */ public class GreenplumSinkFactory extends JdbcSinkFactory { public GreenplumSinkFactory(SyncConf syncConf) { diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/Main.java b/chunjun-core/src/main/java/com/dtstack/chunjun/Main.java index 34963b12fc..5ecf35718d 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/Main.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/Main.java @@ -52,6 +52,7 @@ import com.dtstack.chunjun.util.TableUtil; import org.apache.flink.api.common.JobExecutionResult; +import org.apache.flink.api.common.RuntimeExecutionMode; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.core.execution.JobClient; import org.apache.flink.runtime.jobgraph.SavepointConfigOptions; @@ -143,6 +144,8 @@ private static void exeSqlJob( try { configStreamExecutionEnvironment(env, options, null); List jarUrlList = ExecuteProcessHelper.getExternalJarUrls(options.getAddjar()); + String runMode = options.getRunMode(); + if ("batch".equalsIgnoreCase(runMode)) env.setRuntimeMode(RuntimeExecutionMode.BATCH); StatementSet statementSet = SqlParser.parseSql(job, jarUrlList, tableEnv); TableResult execute = statementSet.execute(); if (env instanceof MyLocalStreamEnvironment) { diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/options/Options.java b/chunjun-core/src/main/java/com/dtstack/chunjun/options/Options.java index de82f1bc86..ef27544de5 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/options/Options.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/options/Options.java @@ -88,6 +88,9 @@ public class Options { @OptionRequired(description = "file add to ship file") private String addShipfile; + @OptionRequired(description = "flink run mode") + private String runMode; + private Configuration flinkConfiguration = null; public Configuration loadFlinkConfiguration() { @@ -235,6 +238,14 @@ public void setJobType(String jobType) { this.jobType = jobType; } + public String getRunMode() { + return runMode; + } + + public void setRunMode(String runMode) { + this.runMode = runMode; + } + @Override public String toString() { return new StringJoiner(", ", Options.class.getSimpleName() + "[", "]") @@ -248,10 +259,12 @@ public String toString() { .add("flinkLibDir='" + flinkLibDir + "'") .add("confProp='" + confProp + "'") .add("p='" + p + "'") + .add("pj='" + pj + "'") .add("pluginLoadMode='" + pluginLoadMode + "'") .add("remoteChunJunDistDir='" + remoteChunJunDistDir + "'") .add("addjar='" + addjar + "'") .add("addShipfile='" + addShipfile + "'") + .add("runMode='" + runMode + "'") .add("flinkConfiguration=" + flinkConfiguration) .toString(); } From 9d5dd68252e568c2d50f90eab20f6e562ca8dfb2 Mon Sep 17 00:00:00 2001 From: kino <56473938+KinoMin@users.noreply.github.com> Date: Fri, 17 Feb 2023 16:03:03 +0800 Subject: [PATCH 32/41] [bug][http] commonConf NullPointerException #1510 (#1511) (cherry picked from commit 8179a6c9dd55376c20523e792baa4553c6689dd5) --- .../chunjun/connector/http/converter/HttpColumnConverter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/converter/HttpColumnConverter.java b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/converter/HttpColumnConverter.java index 729fc668dd..87dbcd8299 100644 --- a/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/converter/HttpColumnConverter.java +++ b/chunjun-connectors/chunjun-connector-http/src/main/java/com/dtstack/chunjun/connector/http/converter/HttpColumnConverter.java @@ -55,6 +55,7 @@ public class HttpColumnConverter public HttpColumnConverter(HttpRestConfig httpRestConfig) { this.httpRestConfig = httpRestConfig; + this.commonConf = httpRestConfig; // Only json need to extract the fields if (StringUtils.isNotBlank((httpRestConfig.getFields()))) { From e72162889d76fef32ebb5f8511ab19747e5f22ec Mon Sep 17 00:00:00 2001 From: OT-QX <43375919+ll076110@users.noreply.github.com> Date: Fri, 17 Feb 2023 16:05:39 +0800 Subject: [PATCH 33/41] [hotfix-#1512][mongodb] Fix the bug that mongodb supports constants (#1513) Co-authored-by: OT-XY (cherry picked from commit a26621f22479c3f680252786791df8d2dac38415) --- .../mongodb/converter/MongodbColumnConverter.java | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/chunjun-connectors/chunjun-connector-mongodb/src/main/java/com/dtstack/chunjun/connector/mongodb/converter/MongodbColumnConverter.java b/chunjun-connectors/chunjun-connector-mongodb/src/main/java/com/dtstack/chunjun/connector/mongodb/converter/MongodbColumnConverter.java index c54af13d02..6a13a9f3a8 100644 --- a/chunjun-connectors/chunjun-connector-mongodb/src/main/java/com/dtstack/chunjun/connector/mongodb/converter/MongodbColumnConverter.java +++ b/chunjun-connectors/chunjun-connector-mongodb/src/main/java/com/dtstack/chunjun/connector/mongodb/converter/MongodbColumnConverter.java @@ -101,17 +101,13 @@ protected MongoSerializationConverter wrapIntoNullableMongodbExternalConverter( public RowData toInternal(Document document) { List fieldList = commonConf.getColumn(); ColumnRowData result = new ColumnRowData(fieldList.size()); - int convertIndex = 0; - for (FieldConf fieldConf : fieldList) { + for (int i = 0; i < fieldList.size(); i++) { AbstractBaseColumn baseColumn = null; - if (StringUtils.isNullOrWhitespaceOnly(fieldConf.getValue())) { - Object field = document.get(fieldConf.getName()); - baseColumn = - (AbstractBaseColumn) - toInternalConverters.get(convertIndex).deserialize(field); - convertIndex++; + if (StringUtils.isNullOrWhitespaceOnly(fieldList.get(i).getValue())) { + Object field = document.get(fieldList.get(i).getName()); + baseColumn = (AbstractBaseColumn) toInternalConverters.get(i).deserialize(field); } - result.addField(assembleFieldProps(fieldConf, baseColumn)); + result.addField(assembleFieldProps(fieldList.get(i), baseColumn)); } return result; } From 88a068ef7d002d6fc0af6d06aec5a852ed18e68c Mon Sep 17 00:00:00 2001 From: zhourui999 <40812600+zhourui999@users.noreply.github.com> Date: Fri, 17 Feb 2023 16:06:21 +0800 Subject: [PATCH 34/41] [hotfix-#1501]fix throw NoRestartException can not sotp application (#1507) (cherry picked from commit ecd926a625de931c3b32cee79e01c3b3e8c82f5f) --- .../com/dtstack/chunjun/sink/format/BaseRichOutputFormat.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/sink/format/BaseRichOutputFormat.java b/chunjun-core/src/main/java/com/dtstack/chunjun/sink/format/BaseRichOutputFormat.java index fdd985796f..b9ea5a2639 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/sink/format/BaseRichOutputFormat.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/sink/format/BaseRichOutputFormat.java @@ -551,8 +551,12 @@ public synchronized FormatState getFormatState() throws Exception { "getFormatState:Start preCommit, rowsOfCurrentTransaction: {}", rowsOfCurrentTransaction); preCommit(); + checkTimerWriteException(); } catch (Exception e) { LOG.error("preCommit error, e = {}", ExceptionUtil.getErrorMessage(e)); + if (e instanceof NoRestartException) { + throw e; + } } finally { flushEnable.compareAndSet(true, false); } From 152b05ce55e3238b46d91de33509c0d34bdf806b Mon Sep 17 00:00:00 2001 From: liuliu Date: Mon, 20 Mar 2023 14:37:15 +0800 Subject: [PATCH 35/41] [hotfix-#1553][jdbc]Fixed an issue where EndLocation was not set when the amount of readable data was 0 --- .../chunjun/connector/jdbc/source/JdbcInputFormat.java | 3 ++- .../com/dtstack/chunjun/metrics/BigIntegerAccumulator.java | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/source/JdbcInputFormat.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/source/JdbcInputFormat.java index 06a30b27f9..1690444439 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/source/JdbcInputFormat.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/source/JdbcInputFormat.java @@ -271,7 +271,8 @@ protected void initMetric(InputSplit inputSplit) { if (StringUtils.isNotBlank(startLocation)) { startLocationAccumulator.add(new BigInteger(startLocation)); // 防止数据库无增量数据时下次从prometheus获取到的startLocation为空 - if (endLocationAccumulator.getLocalValue().longValue() == Long.MIN_VALUE) { + if (endLocationAccumulator.getLocalValue().intValue() + == BigIntegerAccumulator.MIN_VAL) { endLocationAccumulator.add(new BigInteger(startLocation)); } } diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/metrics/BigIntegerAccumulator.java b/chunjun-core/src/main/java/com/dtstack/chunjun/metrics/BigIntegerAccumulator.java index ee163e23d1..003e5b1b4f 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/metrics/BigIntegerAccumulator.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/metrics/BigIntegerAccumulator.java @@ -31,7 +31,9 @@ public class BigIntegerAccumulator implements SimpleAccumulator { private static final long serialVersionUID = 1L; - private BigInteger max = BigInteger.valueOf(Integer.MIN_VALUE); + public static Integer MIN_VAL = Integer.MIN_VALUE; + + private BigInteger max = BigInteger.valueOf(MIN_VAL); public BigIntegerAccumulator() {} From af5c3bbfc8b78ecc00f9011bd49d124fcdef2d49 Mon Sep 17 00:00:00 2001 From: Paddy Gu Date: Thu, 18 May 2023 16:11:43 +0800 Subject: [PATCH 36/41] [Improve][JDBC]jdbcOutputFormat use StatementWrapper (#1374) --- .../clickhouse/dialect/ClickhouseDialect.java | 13 +- .../connector/db2/dialect/Db2Dialect.java | 15 +- .../connector/dm/dialect/DmDialect.java | 5 + .../doris/sink/DorisDynamicTableSink.java | 1 + .../doris/sink/DorisSinkFactory.java | 3 +- .../greenplum/dialect/GreenplumDialect.java | 5 + .../chunjun/connector/jdbc/conf/JdbcConf.java | 3 +- .../connector/jdbc/dialect/JdbcDialect.java | 69 ++- .../jdbc/sink/DynamicPreparedStmt.java | 209 ---------- .../jdbc/sink/JdbcDynamicTableSink.java | 25 +- .../connector/jdbc/sink/JdbcOutputFormat.java | 194 +++++++-- .../jdbc/sink/JdbcOutputFormatBuilder.java | 10 + .../connector/jdbc/sink/JdbcSinkFactory.java | 14 +- .../jdbc/sink/PreparedStmtProxy.java | 393 ------------------ .../InsertOrUpdateStatementWrapper.java | 132 ++++++ .../wrapper/JdbcBatchStatementWrapper.java | 74 ++++ .../sink/wrapper/SimpleStatementWrapper.java | 85 ++++ .../sink/wrapper/StatementWrapperUtil.java | 268 ++++++++++++ .../buffer/InsertDeleteCompactionWrapper.java | 131 ++++++ .../InsertOrUpdateCompactionWrapper.java | 111 +++++ .../NoKeyInsertDeleteCompactionWrapper.java | 129 ++++++ .../buffer/UpsertDeleteCompactionWrapper.java | 116 ++++++ .../wrapper/proxy/CachedWrapperProxy.java | 145 +++++++ .../proxy/DynamicSimpleDeleteWrapper.java | 119 ++++++ .../proxy/NoKeyDeleteWrapperProxy.java | 87 ++++ .../wrapper/proxy/RestoreWrapperProxy.java | 88 ++++ .../jdbc/source/JdbcSourceFactory.java | 2 +- .../FieldNamedPreparedStatement.java | 5 +- .../FieldNamedPreparedStatementImpl.java | 61 ++- .../chunjun/connector/jdbc/util/JdbcUtil.java | 152 ++++++- .../jdbc/dialect/JdbcDialectTest.java | 7 +- .../jdbc/sink/JdbcOutputFormatTest.java | 33 +- .../connector/mysql/dialect/MysqlDialect.java | 11 +- .../oracle/dialect/OracleDialect.java | 5 + .../postgresql/dialect/PostgresqlDialect.java | 5 + .../saphana/dialect/SaphanaDialect.java | 5 + .../sqlserver/dialect/SqlserverDialect.java | 5 + .../converter/SybaseColumnConverter.java | 8 +- .../vertica11/sink/Vertica11OutputFormat.java | 36 +- .../chunjun/element/ColumnRowData.java | 67 +++ .../sink/format/BaseRichOutputFormat.java | 2 +- .../sql/starrocks/stream_starrocks.sql | 2 +- 42 files changed, 2112 insertions(+), 738 deletions(-) delete mode 100644 chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/DynamicPreparedStmt.java delete mode 100644 chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/PreparedStmtProxy.java create mode 100644 chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/InsertOrUpdateStatementWrapper.java create mode 100644 chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/JdbcBatchStatementWrapper.java create mode 100644 chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/SimpleStatementWrapper.java create mode 100644 chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/StatementWrapperUtil.java create mode 100644 chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/buffer/InsertDeleteCompactionWrapper.java create mode 100644 chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/buffer/InsertOrUpdateCompactionWrapper.java create mode 100644 chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/buffer/NoKeyInsertDeleteCompactionWrapper.java create mode 100644 chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/buffer/UpsertDeleteCompactionWrapper.java create mode 100644 chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/proxy/CachedWrapperProxy.java create mode 100644 chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/proxy/DynamicSimpleDeleteWrapper.java create mode 100644 chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/proxy/NoKeyDeleteWrapperProxy.java create mode 100644 chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/proxy/RestoreWrapperProxy.java diff --git a/chunjun-connectors/chunjun-connector-clickhouse/src/main/java/com/dtstack/chunjun/connector/clickhouse/dialect/ClickhouseDialect.java b/chunjun-connectors/chunjun-connector-clickhouse/src/main/java/com/dtstack/chunjun/connector/clickhouse/dialect/ClickhouseDialect.java index bd97eaccfa..c0a9b222f2 100644 --- a/chunjun-connectors/chunjun-connector-clickhouse/src/main/java/com/dtstack/chunjun/connector/clickhouse/dialect/ClickhouseDialect.java +++ b/chunjun-connectors/chunjun-connector-clickhouse/src/main/java/com/dtstack/chunjun/connector/clickhouse/dialect/ClickhouseDialect.java @@ -24,6 +24,7 @@ import com.dtstack.chunjun.converter.RawTypeConverter; import com.dtstack.chunjun.throwable.ChunJunRuntimeException; +import java.util.List; import java.util.Optional; /** @@ -66,7 +67,17 @@ public Optional getReplaceStatement( } @Override - public String getDeleteStatement(String schema, String tableName, String[] conditionFields) { + public String getKeyedDeleteStatement( + String schema, String tableName, List conditionFieldList) { + throw new ChunJunRuntimeException("Clickhouse does not support delete sql"); + } + + @Override + public String getDeleteStatement( + String schema, + String tableName, + String[] conditionFields, + String[] nullConditionFields) { throw new ChunJunRuntimeException("Clickhouse does not support delete sql"); } diff --git a/chunjun-connectors/chunjun-connector-db2/src/main/java/com/dtstack/chunjun/connector/db2/dialect/Db2Dialect.java b/chunjun-connectors/chunjun-connector-db2/src/main/java/com/dtstack/chunjun/connector/db2/dialect/Db2Dialect.java index 5a058642a6..eacbf2cb47 100644 --- a/chunjun-connectors/chunjun-connector-db2/src/main/java/com/dtstack/chunjun/connector/db2/dialect/Db2Dialect.java +++ b/chunjun-connectors/chunjun-connector-db2/src/main/java/com/dtstack/chunjun/connector/db2/dialect/Db2Dialect.java @@ -21,7 +21,6 @@ import com.dtstack.chunjun.connector.db2.converter.Db2ColumnConverter; import com.dtstack.chunjun.connector.db2.converter.Db2RawTypeConverter; import com.dtstack.chunjun.connector.db2.converter.Db2RowConverter; -import com.dtstack.chunjun.connector.jdbc.conf.JdbcConf; import com.dtstack.chunjun.connector.jdbc.dialect.JdbcDialect; import com.dtstack.chunjun.connector.jdbc.statement.FieldNamedPreparedStatement; import com.dtstack.chunjun.converter.AbstractRowConverter; @@ -39,7 +38,6 @@ import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.function.Function; import java.util.stream.Collectors; /** @@ -137,6 +135,11 @@ public Optional getUpsertStatement( return Optional.of(mergeIntoSql.toString()); } + @Override + public boolean supportUpsert() { + return true; + } + /** build T1."A"=T2."A" or T1."A"=nvl(T2."A",T1."A") */ private String buildUpdateConnection( String[] fieldNames, String[] uniqueKeyFields, boolean allReplace) { @@ -181,11 +184,7 @@ private String buildEqualConditions(String[] uniqueKeyFields) { } @Override - public Function> getTableIdentify() { - return conf -> - Tuple3.of( - null, - StringUtils.upperCase(conf.getSchema()), - StringUtils.upperCase(conf.getTable())); + public Tuple3 getTableIdentify(String confSchema, String confTable) { + return Tuple3.of(null, StringUtils.upperCase(confSchema), StringUtils.upperCase(confTable)); } } diff --git a/chunjun-connectors/chunjun-connector-dm/src/main/java/com/dtstack/chunjun/connector/dm/dialect/DmDialect.java b/chunjun-connectors/chunjun-connector-dm/src/main/java/com/dtstack/chunjun/connector/dm/dialect/DmDialect.java index 8dacbd6bbe..9d572159d9 100644 --- a/chunjun-connectors/chunjun-connector-dm/src/main/java/com/dtstack/chunjun/connector/dm/dialect/DmDialect.java +++ b/chunjun-connectors/chunjun-connector-dm/src/main/java/com/dtstack/chunjun/connector/dm/dialect/DmDialect.java @@ -79,6 +79,11 @@ public String buildDualQueryStatement(String[] column) { return sb.toString(); } + @Override + public boolean supportUpsert() { + return true; + } + @Override public Optional getUpsertStatement( String schema, diff --git a/chunjun-connectors/chunjun-connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/sink/DorisDynamicTableSink.java b/chunjun-connectors/chunjun-connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/sink/DorisDynamicTableSink.java index 7d0a400402..94a879b21f 100644 --- a/chunjun-connectors/chunjun-connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/sink/DorisDynamicTableSink.java +++ b/chunjun-connectors/chunjun-connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/sink/DorisDynamicTableSink.java @@ -117,6 +117,7 @@ private DorisJdbcOutputFormatBuilder jdbcBuilder(RowType rowType, DorisConf dori builder.setJdbcDialect(jdbcDialect); builder.setJdbcConf(jdbcConf); builder.setRowConverter(new DorisJdbcRowConverter(rowType)); + setKeyRowConverter(builder, rowType); return builder; } diff --git a/chunjun-connectors/chunjun-connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/sink/DorisSinkFactory.java b/chunjun-connectors/chunjun-connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/sink/DorisSinkFactory.java index 8000aaaea1..ae0dc1fe38 100644 --- a/chunjun-connectors/chunjun-connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/sink/DorisSinkFactory.java +++ b/chunjun-connectors/chunjun-connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/sink/DorisSinkFactory.java @@ -176,7 +176,8 @@ protected void initColumnInfo( Connection conn = JdbcUtil.getConnection(conf, dialect); // get table metadata - Tuple3 tableIdentify = dialect.getTableIdentify().apply(conf); + Tuple3 tableIdentify = + dialect.getTableIdentify(conf.getSchema(), conf.getTable()); Pair, List> tableMetaData = JdbcUtil.getTableMetaData( tableIdentify.f0, tableIdentify.f1, tableIdentify.f2, conn); diff --git a/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/dialect/GreenplumDialect.java b/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/dialect/GreenplumDialect.java index 56b2e64ca2..536dd1f01b 100644 --- a/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/dialect/GreenplumDialect.java +++ b/chunjun-connectors/chunjun-connector-greenplum/src/main/java/com/dtstack/chunjun/connector/greenplum/dialect/GreenplumDialect.java @@ -51,6 +51,11 @@ public Optional defaultDriverName() { return Optional.of(DRIVER); } + @Override + public boolean supportUpsert() { + return false; + } + @Override public Optional getUpsertStatement( String schema, diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/conf/JdbcConf.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/conf/JdbcConf.java index a3f4ca1b38..6156284152 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/conf/JdbcConf.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/conf/JdbcConf.java @@ -23,6 +23,7 @@ import org.apache.commons.lang3.StringUtils; import java.io.Serializable; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Properties; @@ -100,7 +101,7 @@ public class JdbcConf extends ChunJunCommonConf implements Serializable { protected List preSql; protected List postSql; - protected List uniqueKey; + protected List uniqueKey = new ArrayList<>(); @Deprecated protected Map> updateKey; /** upsert 写数据库时，是否null覆盖原来的值 */ diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/dialect/JdbcDialect.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/dialect/JdbcDialect.java index b6ed7e5435..a4d46cbf98 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/dialect/JdbcDialect.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/dialect/JdbcDialect.java @@ -19,7 +19,6 @@ package com.dtstack.chunjun.connector.jdbc.dialect; import com.dtstack.chunjun.conf.ChunJunCommonConf; -import com.dtstack.chunjun.connector.jdbc.conf.JdbcConf; import com.dtstack.chunjun.connector.jdbc.converter.JdbcColumnConverter; import com.dtstack.chunjun.connector.jdbc.converter.JdbcRowConverter; import com.dtstack.chunjun.connector.jdbc.source.JdbcInputSplit; @@ -46,10 +45,11 @@ import java.io.Serializable; import java.math.BigInteger; import java.sql.ResultSet; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.function.Function; import java.util.stream.Collectors; import static java.lang.String.format; @@ -152,6 +152,10 @@ default Optional getUpsertStatement( return Optional.empty(); } + default boolean supportUpsert() { + return false; + } + default Optional getReplaceStatement( String schema, String tableName, String[] fieldNames) { return Optional.empty(); @@ -217,11 +221,12 @@ default String getUpdateStatement( String schema, String tableName, String[] fieldNames, String[] conditionFields) { String setClause = Arrays.stream(fieldNames) - .map(f -> format("%s = ?", quoteIdentifier(f))) + .filter(f -> !Arrays.asList(conditionFields).contains(f)) + .map(f -> format("%s = :%s", quoteIdentifier(f), f)) .collect(Collectors.joining(", ")); String conditionClause = Arrays.stream(conditionFields) - .map(f -> format("%s = ?", quoteIdentifier(f))) + .map(f -> format("%s = :%s", quoteIdentifier(f), f)) .collect(Collectors.joining(" AND ")); return "UPDATE " + buildTableInfoWithSchema(schema, tableName) @@ -231,21 +236,67 @@ default String getUpdateStatement( + conditionClause; } + default String getKeyedDeleteStatement( + String schema, String tableName, List conditionFieldList) { + String conditionClause = + conditionFieldList.stream() + .map(f -> format("%s = :%s", quoteIdentifier(f), f)) + .collect(Collectors.joining(" AND ")); + return "DELETE FROM " + + buildTableInfoWithSchema(schema, tableName) + + " WHERE " + + conditionClause; + } + /** * Get delete one row statement by condition fields, default not use limit 1, because limit 1 is * a sql dialect. */ - default String getDeleteStatement(String schema, String tableName, String[] conditionFields) { - String conditionClause = + default String getDeleteStatement( + String schema, + String tableName, + String[] conditionFields, + String[] nullConditionFields) { + ArrayList nullFields = new ArrayList<>(); + nullFields.addAll(Arrays.asList(nullConditionFields)); + + List conditions = Arrays.stream(conditionFields) + .filter(i -> !nullFields.contains(i)) .map(f -> format("%s = :%s", quoteIdentifier(f), f)) - .collect(Collectors.joining(" AND ")); + .collect(Collectors.toList()); + + Arrays.stream(nullConditionFields) + .map(f -> format("%s IS NULL", quoteIdentifier(f))) + .forEach(i -> conditions.add(i)); + + String conditionClause = String.join(" AND ", conditions); return "DELETE FROM " + buildTableInfoWithSchema(schema, tableName) + " WHERE " + conditionClause; } + /** Get select fields statement by condition fields. Default use SELECT. */ + default String getSelectFromStatement(String schema, String tableName, String[] fields) { + if (fields == null || fields.length == 0) { + throw new IllegalArgumentException("fields can not be null or empty"); + } + + String selectExpressions = + Arrays.stream(fields).map(this::quoteIdentifier).collect(Collectors.joining(", ")); + String fieldExpressions = + Arrays.stream(fields) + .map(f -> format("%s = :%s", quoteIdentifier(f), f)) + .collect(Collectors.joining(" AND ")); + return "SELECT " + + selectExpressions + + " FROM " + + buildTableInfoWithSchema(schema, tableName) + + " WHERE " + + fieldExpressions; + } + /** Get select fields statement by condition fields. Default use SELECT. */ default String getSelectFromStatement( String schema, String tableName, String[] selectFields, String[] conditionFields) { @@ -401,7 +452,7 @@ default KeyUtil initKeyUtil(String incrementName, String incremen } /** catalog/schema/table */ - default Function> getTableIdentify() { - return conf -> Tuple3.of(null, conf.getSchema(), conf.getTable()); + default Tuple3 getTableIdentify(String confSchema, String confTable) { + return Tuple3.of(null, confSchema, confTable); } } diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/DynamicPreparedStmt.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/DynamicPreparedStmt.java deleted file mode 100644 index 70b533a00d..0000000000 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/DynamicPreparedStmt.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.dtstack.chunjun.connector.jdbc.sink; - -import com.dtstack.chunjun.conf.FieldConf; -import com.dtstack.chunjun.connector.jdbc.conf.JdbcConf; -import com.dtstack.chunjun.connector.jdbc.dialect.JdbcDialect; -import com.dtstack.chunjun.connector.jdbc.statement.FieldNamedPreparedStatement; -import com.dtstack.chunjun.connector.jdbc.statement.FieldNamedPreparedStatementImpl; -import com.dtstack.chunjun.connector.jdbc.util.JdbcUtil; -import com.dtstack.chunjun.converter.AbstractRowConverter; -import com.dtstack.chunjun.util.TableUtil; - -import org.apache.flink.table.types.logical.RowType; -import org.apache.flink.types.RowKind; - -import org.apache.commons.lang3.tuple.Pair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.Connection; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * base on row data info to build preparedStatement. row data info include rowkind(which is to set - * which sql kind to use ) - * - *

Company: www.dtstack.com - * - * @author xuchao - * @date 2021-12-20 - */ -public class DynamicPreparedStmt { - - private static final Logger LOG = LoggerFactory.getLogger(DynamicPreparedStmt.class); - - protected List columnNameList = new ArrayList<>(); - - protected List columnTypeList = new ArrayList<>(); - - protected transient FieldNamedPreparedStatement fieldNamedPreparedStatement; - protected JdbcConf jdbcConf; - private boolean writeExtInfo; - private JdbcDialect jdbcDialect; - private AbstractRowConverter rowConverter; - - public static DynamicPreparedStmt buildStmt( - Map header, - Set extHeader, - String schemaName, - String tableName, - RowKind rowKind, - Connection connection, - JdbcDialect jdbcDialect, - boolean writeExtInfo) - throws SQLException { - DynamicPreparedStmt dynamicPreparedStmt = new DynamicPreparedStmt(); - - dynamicPreparedStmt.writeExtInfo = writeExtInfo; - dynamicPreparedStmt.jdbcDialect = jdbcDialect; - dynamicPreparedStmt.getColumnNameList(header, extHeader); - dynamicPreparedStmt.getColumnMeta(schemaName, tableName, connection); - dynamicPreparedStmt.buildRowConvert(); - - String sql = dynamicPreparedStmt.prepareTemplates(rowKind, schemaName, tableName); - String[] fieldNames = new String[dynamicPreparedStmt.columnNameList.size()]; - dynamicPreparedStmt.columnNameList.toArray(fieldNames); - dynamicPreparedStmt.fieldNamedPreparedStatement = - FieldNamedPreparedStatementImpl.prepareStatement(connection, sql, fieldNames); - return dynamicPreparedStmt; - } - - public static DynamicPreparedStmt buildStmt( - String schemaName, - String tableName, - RowKind rowKind, - Connection connection, - JdbcDialect jdbcDialect, - List fieldConfList, - AbstractRowConverter rowConverter) - throws SQLException { - DynamicPreparedStmt dynamicPreparedStmt = new DynamicPreparedStmt(); - dynamicPreparedStmt.jdbcDialect = jdbcDialect; - dynamicPreparedStmt.rowConverter = rowConverter; - String[] fieldNames = new String[fieldConfList.size()]; - for (int i = 0; i < fieldConfList.size(); i++) { - FieldConf fieldConf = fieldConfList.get(i); - fieldNames[i] = fieldConf.getName(); - dynamicPreparedStmt.columnNameList.add(fieldConf.getName()); - dynamicPreparedStmt.columnTypeList.add(fieldConf.getType()); - } - String sql = dynamicPreparedStmt.prepareTemplates(rowKind, schemaName, tableName); - dynamicPreparedStmt.fieldNamedPreparedStatement = - FieldNamedPreparedStatementImpl.prepareStatement(connection, sql, fieldNames); - return dynamicPreparedStmt; - } - - public static DynamicPreparedStmt buildStmt( - JdbcDialect jdbcDialect, - List fieldConfList, - AbstractRowConverter rowConverter, - FieldNamedPreparedStatement fieldNamedPreparedStatement) { - DynamicPreparedStmt dynamicPreparedStmt = new DynamicPreparedStmt(); - dynamicPreparedStmt.jdbcDialect = jdbcDialect; - dynamicPreparedStmt.rowConverter = rowConverter; - dynamicPreparedStmt.fieldNamedPreparedStatement = fieldNamedPreparedStatement; - for (int i = 0; i < fieldConfList.size(); i++) { - FieldConf fieldConf = fieldConfList.get(i); - dynamicPreparedStmt.columnNameList.add(fieldConf.getName()); - dynamicPreparedStmt.columnTypeList.add(fieldConf.getType()); - } - return dynamicPreparedStmt; - } - - protected String prepareTemplates(RowKind rowKind, String schemaName, String tableName) { - String singleSql = null; - switch (rowKind) { - case INSERT: - case UPDATE_AFTER: - singleSql = - jdbcDialect.getInsertIntoStatement( - schemaName, tableName, columnNameList.toArray(new String[0])); - break; - case DELETE: - case UPDATE_BEFORE: - String[] columnNames = new String[columnNameList.size()]; - columnNameList.toArray(columnNames); - singleSql = jdbcDialect.getDeleteStatement(schemaName, tableName, columnNames); - break; - default: - // TODO 异常如何处理 - LOG.warn("not support RowKind: {}", rowKind); - } - - return singleSql; - } - - public void getColumnNameList(Map header, Set extHeader) { - if (writeExtInfo) { - columnNameList.addAll(header.keySet()); - } else { - header.keySet().stream() - .filter(fieldName -> !extHeader.contains(fieldName)) - .forEach(fieldName -> columnNameList.add(fieldName)); - } - } - - public void buildRowConvert() { - RowType rowType = - TableUtil.createRowType( - columnNameList, columnTypeList, jdbcDialect.getRawTypeConverter()); - rowConverter = jdbcDialect.getColumnConverter(rowType, jdbcConf); - } - - public void getColumnMeta(String schema, String table, Connection dbConn) { - Pair, List> listListPair = - JdbcUtil.getTableMetaData(null, schema, table, dbConn); - List nameList = listListPair.getLeft(); - List typeList = listListPair.getRight(); - for (String columnName : columnNameList) { - int index = nameList.indexOf(columnName); - columnTypeList.add(typeList.get(index)); - } - } - - public void reOpenStatement(Connection connection) throws SQLException { - this.fieldNamedPreparedStatement.reOpen(connection); - } - - public void close() throws SQLException { - fieldNamedPreparedStatement.close(); - } - - public FieldNamedPreparedStatement getFieldNamedPreparedStatement() { - return fieldNamedPreparedStatement; - } - - public void setFieldNamedPreparedStatement( - FieldNamedPreparedStatement fieldNamedPreparedStatement) { - this.fieldNamedPreparedStatement = fieldNamedPreparedStatement; - } - - public AbstractRowConverter getRowConverter() { - return rowConverter; - } - - public void setRowConverter(AbstractRowConverter rowConverter) { - this.rowConverter = rowConverter; - } -} diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcDynamicTableSink.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcDynamicTableSink.java index 27a876fd34..3c1d10a5ef 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcDynamicTableSink.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcDynamicTableSink.java @@ -21,7 +21,6 @@ import com.dtstack.chunjun.conf.FieldConf; import com.dtstack.chunjun.connector.jdbc.conf.JdbcConf; import com.dtstack.chunjun.connector.jdbc.dialect.JdbcDialect; -import com.dtstack.chunjun.enums.EWriteMode; import com.dtstack.chunjun.sink.DtOutputFormatSinkFunction; import org.apache.flink.table.api.TableSchema; @@ -70,7 +69,6 @@ public ChangelogMode getChangelogMode(ChangelogMode requestedMode) { .addContainedKind(RowKind.INSERT) .addContainedKind(RowKind.DELETE) .addContainedKind(RowKind.UPDATE_AFTER) - .addContainedKind(RowKind.UPDATE_BEFORE) .build(); } @@ -104,10 +102,6 @@ public SinkFunctionProvider getSinkRuntimeProvider(Context context) { columnList.add(field); } jdbcConf.setColumn(columnList); - jdbcConf.setMode( - (CollectionUtil.isNullOrEmpty(jdbcConf.getUniqueKey())) - ? EWriteMode.INSERT.name() - : EWriteMode.UPDATE.name()); builder.setColumnNameList(columnNameList); builder.setColumnTypeList(columnTypeList); @@ -115,11 +109,30 @@ public SinkFunctionProvider getSinkRuntimeProvider(Context context) { builder.setJdbcDialect(jdbcDialect); builder.setJdbcConf(jdbcConf); builder.setRowConverter(jdbcDialect.getRowConverter(rowType)); + setKeyRowConverter(builder, rowType); return SinkFunctionProvider.of( new DtOutputFormatSinkFunction(builder.finish()), jdbcConf.getParallelism()); } + protected void setKeyRowConverter(JdbcOutputFormatBuilder builder, RowType rowType) { + if (!CollectionUtil.isNullOrEmpty(jdbcConf.getUniqueKey())) { + List fields = rowType.getFields(); + List keyRowFields = new ArrayList<>(jdbcConf.getUniqueKey().size()); + for (String name : jdbcConf.getUniqueKey()) { + for (RowType.RowField field : fields) { + if (Objects.equals(name, field.getName())) { + keyRowFields.add(field); + break; + } + } + } + RowType keyRowType = new RowType(keyRowFields); + builder.setKeyRowType(keyRowType); + builder.setKeyRowConverter(jdbcDialect.getRowConverter(keyRowType)); + } + } + @Override public DynamicTableSink copy() { return new JdbcDynamicTableSink(jdbcConf, jdbcDialect, tableSchema, builder); diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormat.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormat.java index 2dd7433c38..b769cb3b12 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormat.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormat.java @@ -23,8 +23,14 @@ import com.dtstack.chunjun.cdc.ddl.definition.TableIdentifier; import com.dtstack.chunjun.connector.jdbc.conf.JdbcConf; import com.dtstack.chunjun.connector.jdbc.dialect.JdbcDialect; +import com.dtstack.chunjun.connector.jdbc.sink.wrapper.InsertOrUpdateStatementWrapper; +import com.dtstack.chunjun.connector.jdbc.sink.wrapper.JdbcBatchStatementWrapper; +import com.dtstack.chunjun.connector.jdbc.sink.wrapper.SimpleStatementWrapper; +import com.dtstack.chunjun.connector.jdbc.sink.wrapper.buffer.UpsertDeleteCompactionWrapper; +import com.dtstack.chunjun.connector.jdbc.sink.wrapper.proxy.RestoreWrapperProxy; import com.dtstack.chunjun.connector.jdbc.statement.FieldNamedPreparedStatement; import com.dtstack.chunjun.connector.jdbc.util.JdbcUtil; +import com.dtstack.chunjun.converter.AbstractRowConverter; import com.dtstack.chunjun.element.ColumnRowData; import com.dtstack.chunjun.enums.EWriteMode; import com.dtstack.chunjun.enums.Semantic; @@ -36,6 +42,7 @@ import org.apache.flink.table.data.GenericRowData; import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.logical.RowType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -47,6 +54,7 @@ import java.sql.Statement; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Set; /** @@ -67,10 +75,13 @@ public class JdbcOutputFormat extends BaseRichOutputFormat { protected transient Connection dbConn; - protected transient PreparedStmtProxy stmtProxy; + protected transient JdbcBatchStatementWrapper statementWrapper; protected Set createTableOnSnapShot = new HashSet<>(); + private AbstractRowConverter keyRowConverter; + private RowType keyRowType; + @Override public void initializeGlobal(int parallelism) { executeBatch(jdbcConf.getPreSql()); @@ -98,7 +109,7 @@ protected void openInternal(int taskNumber, int numTasks) { } } - buildStmtProxy(); + buildStatementWrapper(); LOG.info("subTask[{}}] wait finished", taskNumber); } catch (SQLException sqe) { throw new IllegalArgumentException("open() failed.", sqe); @@ -107,30 +118,95 @@ protected void openInternal(int taskNumber, int numTasks) { } } - public void buildStmtProxy() throws SQLException { + public void buildStatementWrapper() throws SQLException { String tableInfo = jdbcConf.getTable(); if ("*".equalsIgnoreCase(tableInfo)) { - stmtProxy = new PreparedStmtProxy(dbConn, jdbcDialect, false); + // restoration + statementWrapper = new RestoreWrapperProxy(dbConn, jdbcDialect, false); + } else { + if (useAbstractColumn || CollectionUtils.isEmpty(jdbcConf.getUniqueKey())) { + // sync or sql appendOnly + FieldNamedPreparedStatement fieldNamedPreparedStatement = + FieldNamedPreparedStatement.prepareStatement( + dbConn, + prepareTemplates(), + this.columnNameList.toArray(new String[0])); + statementWrapper = + new SimpleStatementWrapper(fieldNamedPreparedStatement, rowConverter); + } else { + // sql retract + buildRetractStatementExecutor(); + } + } + } + + private void buildRetractStatementExecutor() throws SQLException { + SimpleStatementWrapper deleteExecutor = + new SimpleStatementWrapper( + FieldNamedPreparedStatement.prepareStatement( + dbConn, + jdbcDialect.getKeyedDeleteStatement( + jdbcConf.getSchema(), + jdbcConf.getTable(), + jdbcConf.getUniqueKey()), + jdbcConf.getUniqueKey().toArray(new String[0])), + keyRowConverter); + JdbcBatchStatementWrapper upsertExecutor; + if (jdbcDialect.supportUpsert()) { + upsertExecutor = + new SimpleStatementWrapper( + FieldNamedPreparedStatement.prepareStatement( + dbConn, + getUpsertStatement(), + this.columnNameList.toArray(new String[0])), + rowConverter); } else { - FieldNamedPreparedStatement fieldNamedPreparedStatement = - FieldNamedPreparedStatement.prepareStatement( - dbConn, prepareTemplates(), this.columnNameList.toArray(new String[0])); - stmtProxy = - new PreparedStmtProxy( - fieldNamedPreparedStatement, - rowConverter, - dbConn, - jdbcConf, - jdbcDialect); + upsertExecutor = getInsertOrUpdateExecutor(); } + statementWrapper = + new UpsertDeleteCompactionWrapper( + upsertExecutor, + deleteExecutor, + JdbcUtil.getKeyExtractor( + columnNameList, jdbcConf.getUniqueKey(), keyRowType, false)); + } + + private JdbcBatchStatementWrapper getInsertOrUpdateExecutor() throws SQLException { + FieldNamedPreparedStatement insertStatement = + FieldNamedPreparedStatement.prepareStatement( + dbConn, + getInsertPrepareTemplate(), + this.columnNameList.toArray(new String[0])); + FieldNamedPreparedStatement updateStatement = + FieldNamedPreparedStatement.prepareStatement( + dbConn, + getUpdatePrepareTemplate(), + this.columnNameList.toArray(new String[0])); + FieldNamedPreparedStatement selectStatement = + FieldNamedPreparedStatement.prepareStatement( + dbConn, + jdbcDialect.getSelectFromStatement( + jdbcConf.getSchema(), + jdbcConf.getTable(), + jdbcConf.getUniqueKey().toArray(new String[0])), + jdbcConf.getUniqueKey().toArray(new String[0])); + return new InsertOrUpdateStatementWrapper( + insertStatement, + updateStatement, + selectStatement, + JdbcUtil.getKeyExtractor( + columnNameList, jdbcConf.getUniqueKey(), keyRowType, false), + rowConverter, + rowConverter, + keyRowConverter); } @Override protected void writeSingleRecordInternal(RowData row) throws WriteRecordException { int index = 0; try { - stmtProxy.writeSingleRecordInternal(row); + statementWrapper.writeSingleRecord(row); if (Semantic.EXACTLY_ONCE == semantic) { rowsOfCurrentTransaction += rows.size(); } else { @@ -157,11 +233,10 @@ protected String recordConvertDetailErrorMessage(int pos, Object row) { protected void writeMultipleRecordsInternal() throws Exception { try { for (RowData row : rows) { - stmtProxy.convertToExternal(row); - stmtProxy.addBatch(); + statementWrapper.addToBatch(row); lastRow = row; } - stmtProxy.executeBatch(); + statementWrapper.executeBatch(); // 开启了cp，但是并没有使用2pc方式让下游数据可见 if (Semantic.EXACTLY_ONCE == semantic) { rowsOfCurrentTransaction += rows.size(); @@ -178,7 +253,7 @@ protected void writeMultipleRecordsInternal() throws Exception { throw e; } finally { // 执行完后清空batch - stmtProxy.clearBatch(); + statementWrapper.clearBatch(); } } @@ -203,7 +278,7 @@ public void preCommit() throws Exception { if (rows != null && rows.size() > 0) { super.writeRecordInternal(); } else { - stmtProxy.executeBatch(); + statementWrapper.executeBatch(); } } @@ -224,7 +299,7 @@ public void doCommit() throws SQLException { } snapshotWriteCounter.add(rowsOfCurrentTransaction); rowsOfCurrentTransaction = 0; - stmtProxy.clearStatementCache(); + statementWrapper.clearStatementCache(); } catch (Exception e) { dbConn.rollback(); throw e; @@ -258,32 +333,55 @@ protected void executeBatch(List sqlList) { } } + protected String getInsertPrepareTemplate() { + return jdbcDialect.getInsertIntoStatement( + jdbcConf.getSchema(), jdbcConf.getTable(), columnNameList.toArray(new String[0])); + } + + protected String getUpdatePrepareTemplate() { + return jdbcDialect.getUpdateStatement( + jdbcConf.getSchema(), + jdbcConf.getTable(), + columnNameList.toArray(new String[0]), + jdbcConf.getUniqueKey().toArray(new String[0])); + } + + protected String getReplacePrepareTemplate() { + Optional replaceStatement = + jdbcDialect.getReplaceStatement( + jdbcConf.getSchema(), + jdbcConf.getTable(), + columnNameList.toArray(new String[0])); + if (replaceStatement.isPresent()) { + return replaceStatement.get(); + } else { + throw new IllegalArgumentException("Unknown write mode:" + jdbcConf.getMode()); + } + } + + protected String getUpsertStatement() { + Optional upsertStatement = + jdbcDialect.getUpsertStatement( + jdbcConf.getSchema(), + jdbcConf.getTable(), + columnNameList.toArray(new String[0]), + jdbcConf.getUniqueKey().toArray(new String[0]), + jdbcConf.isAllReplace()); + if (upsertStatement.isPresent()) { + return upsertStatement.get(); + } else { + throw new IllegalArgumentException("Unknown write mode:" + jdbcConf.getMode()); + } + } + protected String prepareTemplates() { String singleSql; if (EWriteMode.INSERT.name().equalsIgnoreCase(jdbcConf.getMode())) { - singleSql = - jdbcDialect.getInsertIntoStatement( - jdbcConf.getSchema(), - jdbcConf.getTable(), - columnNameList.toArray(new String[0])); + singleSql = getInsertPrepareTemplate(); } else if (EWriteMode.REPLACE.name().equalsIgnoreCase(jdbcConf.getMode())) { - singleSql = - jdbcDialect - .getReplaceStatement( - jdbcConf.getSchema(), - jdbcConf.getTable(), - columnNameList.toArray(new String[0])) - .get(); + singleSql = getReplacePrepareTemplate(); } else if (EWriteMode.UPDATE.name().equalsIgnoreCase(jdbcConf.getMode())) { - singleSql = - jdbcDialect - .getUpsertStatement( - jdbcConf.getSchema(), - jdbcConf.getTable(), - columnNameList.toArray(new String[0]), - jdbcConf.getUniqueKey().toArray(new String[0]), - jdbcConf.isAllReplace()) - .get(); + singleSql = getUpsertStatement(); } else { throw new IllegalArgumentException("Unknown write mode:" + jdbcConf.getMode()); } @@ -396,8 +494,8 @@ && tableExist( public void closeInternal() { snapshotWriteCounter.add(rowsOfCurrentTransaction); try { - if (stmtProxy != null) { - stmtProxy.close(); + if (statementWrapper != null) { + statementWrapper.close(); } } catch (SQLException e) { LOG.error(ExceptionUtil.getErrorMessage(e)); @@ -442,4 +540,12 @@ public void setColumnNameList(List columnNameList) { public void setColumnTypeList(List columnTypeList) { this.columnTypeList = columnTypeList; } + + public void setKeyRowType(RowType keyRowType) { + this.keyRowType = keyRowType; + } + + public void setKeyRowConverter(AbstractRowConverter keyRowConverter) { + this.keyRowConverter = keyRowConverter; + } } diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormatBuilder.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormatBuilder.java index 7b75737cbd..ed36327926 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormatBuilder.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormatBuilder.java @@ -23,6 +23,8 @@ import com.dtstack.chunjun.enums.Semantic; import com.dtstack.chunjun.sink.format.BaseRichOutputFormatBuilder; +import org.apache.flink.table.types.logical.RowType; + import org.apache.commons.lang3.StringUtils; import java.util.List; @@ -48,6 +50,14 @@ public void setRowConverter(AbstractRowConverter rowConverter) { format.setRowConverter(rowConverter); } + public void setKeyRowType(RowType keyRowType) { + format.setKeyRowType(keyRowType); + } + + public void setKeyRowConverter(AbstractRowConverter keyRowConverter) { + format.setKeyRowConverter(keyRowConverter); + } + public void setColumnNameList(List columnNameList) { format.setColumnNameList(columnNameList); } diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcSinkFactory.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcSinkFactory.java index 9776746f6d..15ae647d4f 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcSinkFactory.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcSinkFactory.java @@ -49,6 +49,7 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.stream.Collectors; public abstract class JdbcSinkFactory extends SinkFactory { @@ -103,15 +104,26 @@ public DataStreamSink createSink(DataStream dataSet) { builder.setColumnTypeList(columnTypeList); AbstractRowConverter rowConverter; + AbstractRowConverter keyRowConverter; final RowType rowType = TableUtil.createRowType(jdbcConf.getColumn(), getRawTypeConverter()); + final RowType keyRowType = + TableUtil.createRowType( + jdbcConf.getColumn().stream() + .filter(field -> jdbcConf.getUniqueKey().contains(field.getName())) + .collect(Collectors.toList()), + getRawTypeConverter()); // 同步任务使用transform if (!useAbstractBaseColumn) { rowConverter = jdbcDialect.getRowConverter(rowType); + keyRowConverter = jdbcDialect.getRowConverter(keyRowType); } else { rowConverter = jdbcDialect.getColumnConverter(rowType, jdbcConf); + keyRowConverter = jdbcDialect.getColumnConverter(keyRowType, jdbcConf); } builder.setRowConverter(rowConverter, useAbstractBaseColumn); + builder.setKeyRowType(keyRowType); + builder.setKeyRowConverter(keyRowConverter); return createOutput(dataSet, builder.finish()); } @@ -128,7 +140,7 @@ protected void initColumnInfo() { protected Pair, List> getTableMetaData(Connection dbConn) { Tuple3 tableIdentify = - jdbcDialect.getTableIdentify().apply(jdbcConf); + jdbcDialect.getTableIdentify(jdbcConf.getSchema(), jdbcConf.getTable()); return JdbcUtil.getTableMetaData( tableIdentify.f0, tableIdentify.f1, tableIdentify.f2, dbConn); } diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/PreparedStmtProxy.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/PreparedStmtProxy.java deleted file mode 100644 index 6ef65e88a4..0000000000 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/PreparedStmtProxy.java +++ /dev/null @@ -1,393 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.dtstack.chunjun.connector.jdbc.sink; - -import com.dtstack.chunjun.connector.jdbc.conf.JdbcConf; -import com.dtstack.chunjun.connector.jdbc.dialect.JdbcDialect; -import com.dtstack.chunjun.connector.jdbc.statement.FieldNamedPreparedStatement; -import com.dtstack.chunjun.constants.CDCConstantValue; -import com.dtstack.chunjun.converter.AbstractRowConverter; -import com.dtstack.chunjun.element.ColumnRowData; - -import org.apache.flink.table.data.RowData; -import org.apache.flink.types.RowKind; - -import com.esotericsoftware.minlog.Log; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import org.apache.commons.collections.MapUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.InputStream; -import java.io.Reader; -import java.math.BigDecimal; -import java.sql.Array; -import java.sql.Connection; -import java.sql.Date; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Time; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; - -/** - * build prepare proxy, proxy implements FieldNamedPreparedStatement. it support to build - * preparestmt and manager it with cache. - * - *

Company: www.dtstack.com - * - * @author xuchao - * @date 2021-12-20 - */ -public class PreparedStmtProxy implements FieldNamedPreparedStatement { - - private static final Logger LOG = LoggerFactory.getLogger(PreparedStmtProxy.class); - - private final int cacheSize = 100; - - private final int cacheDurationMin = 10; - - /** LUR cache key info: database_table_rowkind * */ - protected Cache pstmtCache; - - protected boolean cacheIsExpire = false; - - /** 当前的执行sql的preparestatement */ - protected transient FieldNamedPreparedStatement currentFieldNamedPstmt; - - /** 当前执行sql的数据类型转换器 */ - protected AbstractRowConverter currentRowConverter; - - /** 当调用writeMultipleRecords 可能会涉及到多个pstmt */ - private final Set unExecutePstmt = new LinkedHashSet<>(); - - protected Connection connection; - protected JdbcDialect jdbcDialect; - protected JdbcConf jdbcConf; - - /** 是否将框架额外添加的扩展信息写入到数据库,默认不写入* */ - protected boolean writeExtInfo; - - public PreparedStmtProxy(Connection connection, JdbcDialect jdbcDialect, boolean writeExtInfo) { - this.connection = connection; - this.jdbcDialect = jdbcDialect; - this.writeExtInfo = writeExtInfo; - this.cacheIsExpire = true; - initCache(true); - } - - public PreparedStmtProxy( - FieldNamedPreparedStatement currentFieldNamedPstmt, - AbstractRowConverter currentRowConverter, - Connection connection, - JdbcConf jdbcConf, - JdbcDialect jdbcDialect) { - this.currentFieldNamedPstmt = currentFieldNamedPstmt; - this.currentRowConverter = currentRowConverter; - this.connection = connection; - this.jdbcConf = jdbcConf; - this.jdbcDialect = jdbcDialect; - initCache(false); - this.pstmtCache.put( - getPstmtCacheKey(jdbcConf.getSchema(), jdbcConf.getTable(), RowKind.INSERT), - DynamicPreparedStmt.buildStmt( - jdbcDialect, - jdbcConf.getColumn(), - currentRowConverter, - currentFieldNamedPstmt)); - } - - public void convertToExternal(RowData row) throws Exception { - getOrCreateFieldNamedPstmt(row); - if (!writeExtInfo) { - if (row instanceof ColumnRowData) { - ColumnRowData copy = ((ColumnRowData) row).copy(); - copy.removeExtHeaderInfo(); - row = copy; - } - } - - currentFieldNamedPstmt = - (FieldNamedPreparedStatement) - currentRowConverter.toExternal(row, this.currentFieldNamedPstmt); - } - - public void getOrCreateFieldNamedPstmt(RowData row) throws ExecutionException { - if (row instanceof ColumnRowData) { - ColumnRowData columnRowData = (ColumnRowData) row; - Map head = columnRowData.getHeaderInfo(); - if (MapUtils.isEmpty(head)) { - return; - } - int dataBaseIndex = head.get(CDCConstantValue.SCHEMA); - int tableIndex = head.get(CDCConstantValue.TABLE); - - String database = row.getString(dataBaseIndex).toString(); - String tableName = row.getString(tableIndex).toString(); - String key = getPstmtCacheKey(database, tableName, row.getRowKind()); - - DynamicPreparedStmt fieldNamedPreparedStatement = - pstmtCache.get( - key, - () -> { - try { - return DynamicPreparedStmt.buildStmt( - columnRowData.getHeaderInfo(), - columnRowData.getExtHeader(), - database, - tableName, - columnRowData.getRowKind(), - connection, - jdbcDialect, - writeExtInfo); - } catch (SQLException e) { - LOG.warn("", e); - return null; - } - }); - - currentFieldNamedPstmt = fieldNamedPreparedStatement.getFieldNamedPreparedStatement(); - currentRowConverter = fieldNamedPreparedStatement.getRowConverter(); - } else { - String key = - getPstmtCacheKey(jdbcConf.getSchema(), jdbcConf.getTable(), row.getRowKind()); - DynamicPreparedStmt fieldNamedPreparedStatement = - pstmtCache.get( - key, - () -> { - try { - return DynamicPreparedStmt.buildStmt( - jdbcConf.getSchema(), - jdbcConf.getTable(), - row.getRowKind(), - connection, - jdbcDialect, - jdbcConf.getColumn(), - currentRowConverter); - } catch (SQLException e) { - LOG.warn("", e); - return null; - } - }); - currentFieldNamedPstmt = fieldNamedPreparedStatement.getFieldNamedPreparedStatement(); - } - } - - public void writeSingleRecordInternal(RowData row) throws Exception { - getOrCreateFieldNamedPstmt(row); - currentFieldNamedPstmt = - (FieldNamedPreparedStatement) - currentRowConverter.toExternal(row, this.currentFieldNamedPstmt); - currentFieldNamedPstmt.execute(); - } - - protected void initCache(boolean isExpired) { - CacheBuilder cacheBuilder = - CacheBuilder.newBuilder() - .maximumSize(cacheSize) - .removalListener( - notification -> { - try { - assert notification.getValue() != null; - notification.getValue().close(); - } catch (SQLException e) { - Log.error("", e); - } - }); - if (isExpired) { - cacheBuilder.expireAfterAccess(cacheDurationMin, TimeUnit.MINUTES); - } - this.pstmtCache = cacheBuilder.build(); - } - - public String getPstmtCacheKey(String schema, String table, RowKind rowKind) { - return String.format("%s_%s_%s", schema, table, rowKind); - } - - @Override - public void clearParameters() throws SQLException { - currentFieldNamedPstmt.clearParameters(); - } - - @Override - public ResultSet executeQuery() throws SQLException { - return currentFieldNamedPstmt.executeQuery(); - } - - @Override - public void addBatch() throws SQLException { - currentFieldNamedPstmt.addBatch(); - unExecutePstmt.add(currentFieldNamedPstmt); - } - - @Override - public int[] executeBatch() throws SQLException { - List exeResult = new ArrayList<>(); - for (FieldNamedPreparedStatement pstmt : unExecutePstmt) { - int[] resultArray = pstmt.executeBatch(); - Arrays.stream(resultArray).forEach(exeResult::add); - } - - int[] result = new int[exeResult.size()]; - for (int i = 0; i < exeResult.size(); i++) { - result[i] = exeResult.get(i); - } - return result; - } - - @Override - public void clearBatch() throws SQLException { - for (FieldNamedPreparedStatement pstmt : unExecutePstmt) { - pstmt.clearBatch(); - } - unExecutePstmt.clear(); - } - - @Override - public boolean execute() throws SQLException { - return currentFieldNamedPstmt.execute(); - } - - @Override - public void setNull(int fieldIndex, int sqlType) throws SQLException { - currentFieldNamedPstmt.setNull(fieldIndex, sqlType); - } - - @Override - public void setBoolean(int fieldIndex, boolean x) throws SQLException { - currentFieldNamedPstmt.setBoolean(fieldIndex, x); - } - - @Override - public void setByte(int fieldIndex, byte x) throws SQLException { - currentFieldNamedPstmt.setByte(fieldIndex, x); - } - - @Override - public void setShort(int fieldIndex, short x) throws SQLException { - currentFieldNamedPstmt.setShort(fieldIndex, x); - } - - @Override - public void setInt(int fieldIndex, int x) throws SQLException { - currentFieldNamedPstmt.setInt(fieldIndex, x); - } - - @Override - public void setLong(int fieldIndex, long x) throws SQLException { - currentFieldNamedPstmt.setLong(fieldIndex, x); - } - - @Override - public void setFloat(int fieldIndex, float x) throws SQLException { - currentFieldNamedPstmt.setFloat(fieldIndex, x); - } - - @Override - public void setDouble(int fieldIndex, double x) throws SQLException { - currentFieldNamedPstmt.setDouble(fieldIndex, x); - } - - @Override - public void setBigDecimal(int fieldIndex, BigDecimal x) throws SQLException { - currentFieldNamedPstmt.setBigDecimal(fieldIndex, x); - } - - @Override - public void setString(int fieldIndex, String x) throws SQLException { - currentFieldNamedPstmt.setString(fieldIndex, x); - } - - @Override - public void setBytes(int fieldIndex, byte[] x) throws SQLException { - currentFieldNamedPstmt.setBytes(fieldIndex, x); - } - - @Override - public void setDate(int fieldIndex, Date x) throws SQLException { - currentFieldNamedPstmt.setDate(fieldIndex, x); - } - - @Override - public void setTime(int fieldIndex, Time x) throws SQLException { - currentFieldNamedPstmt.setTime(fieldIndex, x); - } - - @Override - public void setTimestamp(int fieldIndex, Timestamp x) throws SQLException { - currentFieldNamedPstmt.setTimestamp(fieldIndex, x); - } - - @Override - public void setObject(int fieldIndex, Object x) throws SQLException { - currentFieldNamedPstmt.setObject(fieldIndex, x); - } - - @Override - public void setBlob(int fieldIndex, InputStream is) throws SQLException { - currentFieldNamedPstmt.setBlob(fieldIndex, is); - } - - @Override - public void setClob(int fieldIndex, Reader reader) throws SQLException { - currentFieldNamedPstmt.setClob(fieldIndex, reader); - } - - @Override - public void setArray(int fieldIndex, Array array) throws SQLException { - currentFieldNamedPstmt.setArray(fieldIndex, array); - } - - @Override - public void close() throws SQLException { - currentFieldNamedPstmt.close(); - } - - @Override - public void reOpen(Connection connection) throws SQLException { - this.connection = connection; - ConcurrentMap stringDynamicPreparedStmtConcurrentMap = - pstmtCache.asMap(); - initCache(cacheIsExpire); - for (Map.Entry entry : - stringDynamicPreparedStmtConcurrentMap.entrySet()) { - DynamicPreparedStmt value = entry.getValue(); - value.reOpenStatement(connection); - pstmtCache.put(entry.getKey(), value); - currentFieldNamedPstmt = value.getFieldNamedPreparedStatement(); - } - } - - @Override - public Connection getConnection() throws SQLException { - return currentFieldNamedPstmt.getConnection(); - } - - public void clearStatementCache() { - pstmtCache.invalidateAll(); - } -} diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/InsertOrUpdateStatementWrapper.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/InsertOrUpdateStatementWrapper.java new file mode 100644 index 0000000000..0e3adc423a --- /dev/null +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/InsertOrUpdateStatementWrapper.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dtstack.chunjun.connector.jdbc.sink.wrapper; + +import com.dtstack.chunjun.connector.jdbc.statement.FieldNamedPreparedStatement; +import com.dtstack.chunjun.converter.AbstractRowConverter; + +import org.apache.flink.table.data.RowData; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.function.Function; + +public class InsertOrUpdateStatementWrapper implements JdbcBatchStatementWrapper { + + private final FieldNamedPreparedStatement insertStatement; + private final FieldNamedPreparedStatement updateStatement; + private final FieldNamedPreparedStatement existStatement; + + private final Function existFieldExtractor; + + private final AbstractRowConverter insertRowConverter; + private final AbstractRowConverter updateRowConverter; + private final AbstractRowConverter existRowConverter; + + public InsertOrUpdateStatementWrapper( + FieldNamedPreparedStatement insertStatement, + FieldNamedPreparedStatement updateStatement, + FieldNamedPreparedStatement existStatement, + Function existFieldExtractor, + AbstractRowConverter insertRowConverter, + AbstractRowConverter updateRowConverter, + AbstractRowConverter existRowConverter) { + this.insertStatement = insertStatement; + this.updateStatement = updateStatement; + this.existStatement = existStatement; + this.existFieldExtractor = existFieldExtractor; + this.insertRowConverter = insertRowConverter; + this.updateRowConverter = updateRowConverter; + this.existRowConverter = existRowConverter; + } + + @Override + public void addToBatch(RowData record) throws Exception { + if (isExist(record)) { + updateRowConverter.toExternal(record, updateStatement); + updateStatement.addBatch(); + } else { + insertRowConverter.toExternal(record, insertStatement); + insertStatement.addBatch(); + } + } + + private boolean isExist(RowData record) throws Exception { + existRowConverter.toExternal(existFieldExtractor.apply(record), existStatement); + try (ResultSet resultSet = existStatement.executeQuery()) { + return resultSet.next(); + } + } + + @Override + public void executeBatch() throws SQLException { + updateStatement.executeBatch(); + insertStatement.executeBatch(); + } + + @Override + public void writeSingleRecord(RowData record) throws Exception { + if (isExist(record)) { + updateRowConverter.toExternal(record, updateStatement); + updateStatement.execute(); + } else { + insertRowConverter.toExternal(record, insertStatement); + insertStatement.execute(); + } + } + + @Override + public ResultSet executeQuery(RowData record) throws SQLException { + throw new UnsupportedOperationException("executeQuery is not supported"); + } + + @Override + public void clearParameters() throws SQLException {} + + @Override + public void close() throws SQLException { + for (FieldNamedPreparedStatement s : + Arrays.asList(existStatement, insertStatement, updateStatement)) { + if (s != null) { + s.close(); + } + } + } + + @Override + public void clearBatch() throws SQLException { + for (FieldNamedPreparedStatement s : + Arrays.asList(existStatement, insertStatement, updateStatement)) { + if (s != null) { + s.clearBatch(); + } + } + } + + @Override + public void reOpen(Connection connection) throws SQLException { + insertStatement.reOpen(connection); + updateStatement.reOpen(connection); + existStatement.reOpen(connection); + } + + @Override + public void clearStatementCache() {} +} diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/JdbcBatchStatementWrapper.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/JdbcBatchStatementWrapper.java new file mode 100644 index 0000000000..6fac6c21e7 --- /dev/null +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/JdbcBatchStatementWrapper.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dtstack.chunjun.connector.jdbc.sink.wrapper; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; + +public interface JdbcBatchStatementWrapper { + + /** + * add statement to batch + * + * @param record the record to add + */ + void addToBatch(T record) throws Exception; + + /** + * Execute the batch. + * + * @throws SQLException if the batch could not be executed + */ + void executeBatch() throws Exception; + + void writeSingleRecord(T record) throws Exception; + + ResultSet executeQuery(T record) throws Exception; + + /** + * Clear parameters. + * + * @throws SQLException if the parameters could not be cleared + */ + void clearParameters() throws SQLException; + + /** + * Close the statement. + * + * @throws SQLException if the statement could not be closed + */ + void close() throws SQLException; + + /** + * Clear the batch. + * + * @throws SQLException if the batch could not be cleared + */ + void clearBatch() throws SQLException; + + /** + * Reopen the statement. + * + * @throws SQLException if the statement could not be reopened + */ + void reOpen(Connection connection) throws SQLException; + + /** Clear {@link JdbcBatchStatementWrapper} cache if exists. */ + void clearStatementCache(); +} diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/SimpleStatementWrapper.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/SimpleStatementWrapper.java new file mode 100644 index 0000000000..d2deffa8f9 --- /dev/null +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/SimpleStatementWrapper.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dtstack.chunjun.connector.jdbc.sink.wrapper; + +import com.dtstack.chunjun.connector.jdbc.statement.FieldNamedPreparedStatement; +import com.dtstack.chunjun.converter.AbstractRowConverter; + +import org.apache.flink.table.data.RowData; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; + +public class SimpleStatementWrapper implements JdbcBatchStatementWrapper { + + protected FieldNamedPreparedStatement statement; + protected AbstractRowConverter rowConverter; + + public SimpleStatementWrapper( + FieldNamedPreparedStatement statement, AbstractRowConverter converter) { + this.statement = statement; + rowConverter = converter; + } + + @Override + public void addToBatch(RowData record) throws Exception { + rowConverter.toExternal(record, statement); + statement.addBatch(); + } + + @Override + public void executeBatch() throws SQLException { + statement.executeBatch(); + } + + @Override + public void writeSingleRecord(RowData record) throws Exception { + rowConverter.toExternal(record, statement); + statement.execute(); + } + + @Override + public ResultSet executeQuery(RowData record) throws Exception { + rowConverter.toInternal(record); + return statement.executeQuery(); + } + + @Override + public void clearParameters() throws SQLException { + statement.clearParameters(); + } + + @Override + public void close() throws SQLException { + statement.close(); + } + + @Override + public void clearBatch() throws SQLException { + statement.clearBatch(); + } + + @Override + public void reOpen(Connection connection) throws SQLException { + statement.reOpen(connection); + } + + @Override + public void clearStatementCache() {} +} diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/StatementWrapperUtil.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/StatementWrapperUtil.java new file mode 100644 index 0000000000..4bb4ac36cc --- /dev/null +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/StatementWrapperUtil.java @@ -0,0 +1,268 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dtstack.chunjun.connector.jdbc.sink.wrapper; + +import com.dtstack.chunjun.connector.jdbc.dialect.JdbcDialect; +import com.dtstack.chunjun.connector.jdbc.sink.wrapper.buffer.InsertDeleteCompactionWrapper; +import com.dtstack.chunjun.connector.jdbc.sink.wrapper.buffer.NoKeyInsertDeleteCompactionWrapper; +import com.dtstack.chunjun.connector.jdbc.sink.wrapper.proxy.NoKeyDeleteWrapperProxy; +import com.dtstack.chunjun.connector.jdbc.statement.FieldNamedPreparedStatement; +import com.dtstack.chunjun.connector.jdbc.util.JdbcUtil; +import com.dtstack.chunjun.element.ColumnRowData; +import com.dtstack.chunjun.throwable.ChunJunRuntimeException; +import com.dtstack.chunjun.util.GsonUtil; +import com.dtstack.chunjun.util.TableUtil; + +import org.apache.flink.api.java.tuple.Tuple3; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.types.RowKind; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.tuple.Pair; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +public class StatementWrapperUtil { + + /** + * only use for restoreMode + * + * @param connection jdbc conn + * @return {@link NoKeyInsertDeleteCompactionWrapper} or {@link InsertDeleteCompactionWrapper} + */ + public static JdbcBatchStatementWrapper buildInsertDeleteExecutor( + Connection connection, + JdbcDialect jdbcDialect, + String schema, + String table, + ColumnRowData columnRowData) + throws SQLException { + Tuple3 tableIdentify = jdbcDialect.getTableIdentify(schema, table); + String catalog = tableIdentify.f0; + schema = tableIdentify.f1; + table = tableIdentify.f2; + List columnNameList = getSelectedNameColumnList(columnRowData); + List columnTypeList = + getColumnTypeFromMetadata(connection, catalog, schema, table, columnNameList); + Pair, List> primaryKeyInfoPair = + getSelectedPrimaryKeyList( + connection, catalog, schema, table, columnNameList, columnTypeList); + + if (CollectionUtils.isNotEmpty(primaryKeyInfoPair.getLeft())) { + return buildInsertDeleteKeyedExecutor( + connection, + schema, + table, + jdbcDialect, + columnNameList, + columnTypeList, + primaryKeyInfoPair.getLeft(), + primaryKeyInfoPair.getRight()); + } else { + return buildNoKeyInsertDeleteExecutor( + connection, schema, table, jdbcDialect, columnNameList, columnTypeList); + } + } + + public static NoKeyInsertDeleteCompactionWrapper buildNoKeyInsertDeleteExecutor( + Connection connection, + String schema, + String table, + JdbcDialect jdbcDialect, + List columnNameList, + List columnTypeList) + throws SQLException { + SimpleStatementWrapper insertStatementExecutor = + getInsertStatementExecutor( + connection, schema, table, jdbcDialect, columnNameList, columnTypeList); + + NoKeyDeleteWrapperProxy deleteNoKeyStatementExecutor = + new NoKeyDeleteWrapperProxy( + connection, schema, table, jdbcDialect, columnNameList, columnTypeList); + + return new NoKeyInsertDeleteCompactionWrapper( + insertStatementExecutor, deleteNoKeyStatementExecutor); + } + + public static InsertDeleteCompactionWrapper buildInsertDeleteKeyedExecutor( + Connection connection, + String schema, + String table, + JdbcDialect jdbcDialect, + List columnNameList, + List columnTypeList, + List primaryColumnNameList, + List primaryColumnTypeList) + throws SQLException { + SimpleStatementWrapper insertStatementExecutor = + getInsertStatementExecutor( + connection, schema, table, jdbcDialect, columnNameList, columnTypeList); + + String keyedDeleteStatement = + jdbcDialect.getKeyedDeleteStatement(schema, table, primaryColumnNameList); + FieldNamedPreparedStatement fieldNamedPreparedStatement = + FieldNamedPreparedStatement.prepareStatement( + connection, + keyedDeleteStatement, + primaryColumnNameList.toArray(new String[0])); + RowType keyRowType = + TableUtil.createRowType( + primaryColumnNameList, + primaryColumnTypeList, + jdbcDialect.getRawTypeConverter()); + SimpleStatementWrapper deleteStatementExecutor = + new SimpleStatementWrapper( + fieldNamedPreparedStatement, jdbcDialect.getColumnConverter(keyRowType)); + + return new InsertDeleteCompactionWrapper( + insertStatementExecutor, + deleteStatementExecutor, + JdbcUtil.getKeyExtractor(columnNameList, primaryColumnNameList, keyRowType, true)); + } + + public static SimpleStatementWrapper getInsertStatementExecutor( + Connection connection, + String schema, + String table, + JdbcDialect jdbcDialect, + List columnNameList, + List columnTypeList) + throws SQLException { + String insertStatement = + jdbcDialect.getInsertIntoStatement( + schema, table, columnNameList.toArray(new String[0])); + FieldNamedPreparedStatement fieldNamedPreparedStatement = + FieldNamedPreparedStatement.prepareStatement( + connection, insertStatement, columnNameList.toArray(new String[0])); + RowType rowType = + TableUtil.createRowType( + columnNameList, columnTypeList, jdbcDialect.getRawTypeConverter()); + return new SimpleStatementWrapper( + fieldNamedPreparedStatement, jdbcDialect.getColumnConverter(rowType)); + } + + /** only use for restoreMode,get fieldType for selectedColumn */ + public static List getColumnTypeFromMetadata( + Connection connection, + String catalog, + String schema, + String table, + List selectedColumnNameList) { + try { + Pair, List> tableMetaData = + JdbcUtil.getTableMetaData(catalog, schema, table, connection); + List metaColumnNameList = tableMetaData.getLeft(); + List metaColumnTypeList = tableMetaData.getRight(); + if (selectedColumnNameList.size() != metaColumnNameList.size()) { + throw new RuntimeException( + "The length of the field on the source end is inconsistent with that on the sink end, and the sink table fields is " + + GsonUtil.GSON.toJson(metaColumnNameList) + + " and the source table field is " + + GsonUtil.GSON.toJson(selectedColumnNameList)); + } + List columnTypeList = new ArrayList<>(selectedColumnNameList.size()); + for (String columnName : selectedColumnNameList) { + int j = 0; + for (; j < metaColumnNameList.size(); j++) { + if (metaColumnNameList.get(j).equals(columnName)) { + columnTypeList.add(metaColumnTypeList.get(j)); + break; + } + } + if (j >= metaColumnNameList.size()) { + throw new RuntimeException( + "The field " + + columnName + + " on the source end is not found on the sink end, and the sink table fields is " + + GsonUtil.GSON.toJson(metaColumnNameList) + + " and the source table field is " + + GsonUtil.GSON.toJson(selectedColumnNameList)); + } + } + return columnTypeList; + } catch (Exception e) { + throw new ChunJunRuntimeException( + String.format( + "get metadata failed,catalog[%s],schema[%s],table[%s]", + catalog, schema, table), + e); + } + } + + public static Pair, List> getSelectedPrimaryKeyList( + Connection connection, + String catalog, + String schema, + String table, + List selectedColumnNameList, + List selectedColumnTypeList) { + try { + List primaryColumnNameList = new ArrayList<>(); + List primaryColumnTypeList = new ArrayList<>(); + ResultSet resultSet = connection.getMetaData().getPrimaryKeys(catalog, schema, table); + while (resultSet.next()) { + String primaryColumnName = resultSet.getString("COLUMN_NAME"); + for (int i = 0; i < selectedColumnNameList.size(); i++) { + String columnName = selectedColumnNameList.get(i); + String columnType = selectedColumnTypeList.get(i); + if (columnName.equals(primaryColumnName)) { + primaryColumnNameList.add(columnName); + primaryColumnTypeList.add(columnType); + } + } + } + return Pair.of(primaryColumnNameList, primaryColumnTypeList); + } catch (SQLException e) { + throw new ChunJunRuntimeException( + String.format( + "get primary key metadata failed,catalog[%s],schema[%s],table[%s]", + catalog, schema, table), + e); + } + } + + public static List getSelectedNameColumnList(ColumnRowData columnRowData) { + return new ArrayList<>(columnRowData.getHeaderInfo().keySet()); + } + + /** + * Returns true if the row kind is INSERT or UPDATE_AFTER, returns false if the row kind is + * DELETE or UPDATE_BEFORE. + */ + public static boolean changeFlag(RowKind rowKind) { + switch (rowKind) { + case INSERT: + case UPDATE_AFTER: + return true; + case DELETE: + case UPDATE_BEFORE: + return false; + default: + throw new UnsupportedOperationException( + String.format( + "Unknown row kind, the supported row kinds is: INSERT, UPDATE_BEFORE, UPDATE_AFTER," + + " DELETE, but get: %s.", + rowKind)); + } + } +} diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/buffer/InsertDeleteCompactionWrapper.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/buffer/InsertDeleteCompactionWrapper.java new file mode 100644 index 0000000000..46cc3b8e7e --- /dev/null +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/buffer/InsertDeleteCompactionWrapper.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dtstack.chunjun.connector.jdbc.sink.wrapper.buffer; + +import com.dtstack.chunjun.connector.jdbc.sink.wrapper.JdbcBatchStatementWrapper; + +import org.apache.flink.table.data.RowData; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +import static com.dtstack.chunjun.connector.jdbc.sink.wrapper.StatementWrapperUtil.changeFlag; + +public class InsertDeleteCompactionWrapper implements JdbcBatchStatementWrapper { + + protected final JdbcBatchStatementWrapper insertExecutor; + protected final JdbcBatchStatementWrapper deleteExecutor; + private final Function keyExtractor; + + Map insertBuffer = new HashMap<>(); + Set deleteBuffer = new HashSet<>(); + + public InsertDeleteCompactionWrapper( + JdbcBatchStatementWrapper insertExecutor, + JdbcBatchStatementWrapper deleteExecutor, + Function keyExtractor) { + this.insertExecutor = insertExecutor; + this.deleteExecutor = deleteExecutor; + this.keyExtractor = keyExtractor; + } + + @Override + public void addToBatch(RowData record) { + RowData key = keyExtractor.apply(record); + boolean flag = changeFlag(record.getRowKind()); + if (flag) { + insertBuffer.put(key, record); + } else { + if (insertBuffer.containsKey(key)) { + insertBuffer.remove(key); + } else { + deleteBuffer.add(key); + } + } + } + + @Override + public void executeBatch() throws Exception { + if (!deleteBuffer.isEmpty()) { + for (RowData key : deleteBuffer) { + deleteExecutor.addToBatch(key); + } + deleteExecutor.executeBatch(); + deleteBuffer.clear(); + } + if (!insertBuffer.isEmpty()) { + for (RowData record : insertBuffer.values()) { + insertExecutor.addToBatch(record); + } + insertExecutor.executeBatch(); + insertBuffer.clear(); + } + } + + @Override + public void writeSingleRecord(RowData record) throws Exception { + boolean flag = changeFlag(record.getRowKind()); + if (flag) { + insertExecutor.writeSingleRecord(record); + } else { + RowData key = keyExtractor.apply(record); + deleteExecutor.writeSingleRecord(key); + } + } + + @Override + public ResultSet executeQuery(RowData record) throws Exception { + throw new UnsupportedOperationException("executeQuery is not supported"); + } + + @Override + public void clearParameters() throws SQLException { + deleteExecutor.clearParameters(); + insertExecutor.clearParameters(); + } + + @Override + public void close() throws SQLException { + deleteExecutor.close(); + insertExecutor.close(); + } + + @Override + public void clearBatch() throws SQLException { + deleteExecutor.clearBatch(); + insertExecutor.clearBatch(); + } + + @Override + public void reOpen(Connection connection) throws SQLException { + deleteExecutor.reOpen(connection); + insertExecutor.reOpen(connection); + } + + @Override + public void clearStatementCache() { + deleteExecutor.clearStatementCache(); + insertExecutor.clearStatementCache(); + } +} diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/buffer/InsertOrUpdateCompactionWrapper.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/buffer/InsertOrUpdateCompactionWrapper.java new file mode 100644 index 0000000000..f941a149fa --- /dev/null +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/buffer/InsertOrUpdateCompactionWrapper.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dtstack.chunjun.connector.jdbc.sink.wrapper.buffer; + +import com.dtstack.chunjun.connector.jdbc.sink.wrapper.JdbcBatchStatementWrapper; + +import org.apache.flink.table.data.RowData; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + +public class InsertOrUpdateCompactionWrapper implements JdbcBatchStatementWrapper { + + JdbcBatchStatementWrapper insertExecutor; + JdbcBatchStatementWrapper isExistExecutor; + JdbcBatchStatementWrapper updateExecutor; + Function keyExtractor; + Map buffer = new HashMap<>(); + + @Override + public void addToBatch(RowData record) throws Exception { + RowData key = keyExtractor.apply(record); + buffer.put(key, record); + } + + @Override + public void executeBatch() throws Exception { + if (!buffer.isEmpty()) { + for (Map.Entry entry : buffer.entrySet()) { + RowData key = entry.getKey(); + RowData record = entry.getValue(); + if (isExist(key)) { + updateExecutor.addToBatch(record); + } else { + insertExecutor.addToBatch(record); + } + } + insertExecutor.executeBatch(); + updateExecutor.executeBatch(); + } + } + + private boolean isExist(RowData key) throws Exception { + ResultSet resultSet = isExistExecutor.executeQuery(key); + return resultSet.next(); + } + + @Override + public void writeSingleRecord(RowData record) throws Exception { + if (isExist(record)) { + updateExecutor.writeSingleRecord(record); + } else { + insertExecutor.writeSingleRecord(record); + } + } + + @Override + public ResultSet executeQuery(RowData record) throws SQLException { + throw new UnsupportedOperationException("executeQuery is not supported"); + } + + @Override + public void clearParameters() throws SQLException { + insertExecutor.clearParameters(); + isExistExecutor.clearParameters(); + updateExecutor.clearParameters(); + } + + @Override + public void close() throws SQLException { + insertExecutor.close(); + isExistExecutor.close(); + updateExecutor.close(); + } + + @Override + public void clearBatch() throws SQLException { + insertExecutor.clearBatch(); + isExistExecutor.clearBatch(); + updateExecutor.clearBatch(); + } + + @Override + public void reOpen(Connection connection) throws SQLException { + insertExecutor.reOpen(connection); + isExistExecutor.reOpen(connection); + updateExecutor.reOpen(connection); + } + + @Override + public void clearStatementCache() {} +} diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/buffer/NoKeyInsertDeleteCompactionWrapper.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/buffer/NoKeyInsertDeleteCompactionWrapper.java new file mode 100644 index 0000000000..35fc314dbd --- /dev/null +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/buffer/NoKeyInsertDeleteCompactionWrapper.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dtstack.chunjun.connector.jdbc.sink.wrapper.buffer; + +import com.dtstack.chunjun.connector.jdbc.sink.wrapper.JdbcBatchStatementWrapper; + +import org.apache.flink.table.data.RowData; +import org.apache.flink.types.RowKind; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static com.dtstack.chunjun.connector.jdbc.sink.wrapper.StatementWrapperUtil.changeFlag; + +public class NoKeyInsertDeleteCompactionWrapper implements JdbcBatchStatementWrapper { + protected final JdbcBatchStatementWrapper insertExecutor; + protected final JdbcBatchStatementWrapper deleteExecutor; + protected final Map insertBuffer = new HashMap<>(); + protected final Set deleteBuffer = new HashSet<>(); + + public NoKeyInsertDeleteCompactionWrapper( + JdbcBatchStatementWrapper insertExecutor, + JdbcBatchStatementWrapper deleteExecutor) { + this.insertExecutor = insertExecutor; + this.deleteExecutor = deleteExecutor; + } + + @Override + public void addToBatch(RowData record) { + boolean flag = changeFlag(record.getRowKind()); + // There is no keyExtractor and Changelog will not be used later, For the convenience of + // using all set to INSERT + record.setRowKind(RowKind.INSERT); + if (flag) { + insertBuffer.compute(record, (rowData, time) -> time == null ? 1 : time + 1); + } else { + if (insertBuffer.containsKey(record)) { + insertBuffer.remove(record); + } else { + deleteBuffer.add(record); + } + } + } + + @Override + public void executeBatch() throws Exception { + if (!deleteBuffer.isEmpty()) { + for (RowData record : deleteBuffer) { + deleteExecutor.addToBatch(record); + } + deleteExecutor.executeBatch(); + deleteBuffer.clear(); + } + if (!insertBuffer.isEmpty()) { + for (Map.Entry entry : insertBuffer.entrySet()) { + for (int i = 0; i < entry.getValue(); i++) { + insertExecutor.addToBatch(entry.getKey()); + } + } + insertExecutor.executeBatch(); + insertBuffer.clear(); + } + } + + @Override + public void writeSingleRecord(RowData record) throws Exception { + boolean flag = changeFlag(record.getRowKind()); + if (flag) { + insertExecutor.writeSingleRecord(record); + } else { + deleteExecutor.writeSingleRecord(record); + } + } + + @Override + public ResultSet executeQuery(RowData record) throws SQLException { + throw new UnsupportedOperationException("executeQuery is not supported"); + } + + @Override + public void clearParameters() throws SQLException { + deleteExecutor.clearParameters(); + insertExecutor.clearParameters(); + } + + @Override + public void close() throws SQLException { + deleteExecutor.close(); + insertExecutor.close(); + } + + @Override + public void clearBatch() throws SQLException { + deleteExecutor.clearBatch(); + insertExecutor.clearBatch(); + } + + @Override + public void reOpen(Connection connection) throws SQLException { + deleteExecutor.reOpen(connection); + insertExecutor.reOpen(connection); + } + + @Override + public void clearStatementCache() { + deleteExecutor.clearStatementCache(); + insertExecutor.clearStatementCache(); + } +} diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/buffer/UpsertDeleteCompactionWrapper.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/buffer/UpsertDeleteCompactionWrapper.java new file mode 100644 index 0000000000..979e8fc88b --- /dev/null +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/buffer/UpsertDeleteCompactionWrapper.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dtstack.chunjun.connector.jdbc.sink.wrapper.buffer; + +import com.dtstack.chunjun.connector.jdbc.sink.wrapper.JdbcBatchStatementWrapper; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.table.data.RowData; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + +import static com.dtstack.chunjun.connector.jdbc.sink.wrapper.StatementWrapperUtil.changeFlag; + +public class UpsertDeleteCompactionWrapper implements JdbcBatchStatementWrapper { + + private final JdbcBatchStatementWrapper upsertExecutor; + private final JdbcBatchStatementWrapper deleteExecutor; + private final Function keyExtractor; + + Map> buffer = new HashMap<>(); + + public UpsertDeleteCompactionWrapper( + JdbcBatchStatementWrapper upsertExecutor, + JdbcBatchStatementWrapper deleteExecutor, + Function keyExtractor) { + this.upsertExecutor = upsertExecutor; + this.deleteExecutor = deleteExecutor; + this.keyExtractor = keyExtractor; + } + + @Override + public void addToBatch(RowData record) { + RowData key = keyExtractor.apply(record); + boolean flag = changeFlag(record.getRowKind()); + buffer.put(key, Tuple2.of(flag, record)); + } + + @Override + public void executeBatch() throws Exception { + if (!buffer.isEmpty()) { + for (Map.Entry> entry : buffer.entrySet()) { + if (entry.getValue().f0) { + upsertExecutor.addToBatch(entry.getValue().f1); + } else { + // delete by key + deleteExecutor.addToBatch(entry.getKey()); + } + } + deleteExecutor.executeBatch(); + upsertExecutor.executeBatch(); + buffer.clear(); + } + } + + @Override + public void writeSingleRecord(RowData record) throws Exception { + boolean flag = changeFlag(record.getRowKind()); + if (flag) { + upsertExecutor.writeSingleRecord(record); + } else { + deleteExecutor.writeSingleRecord(keyExtractor.apply(record)); + } + } + + @Override + public ResultSet executeQuery(RowData record) throws SQLException { + throw new UnsupportedOperationException("executeQuery is not supported"); + } + + @Override + public void clearParameters() throws SQLException { + upsertExecutor.clearParameters(); + deleteExecutor.clearParameters(); + } + + @Override + public void close() throws SQLException { + upsertExecutor.close(); + deleteExecutor.close(); + } + + @Override + public void clearBatch() throws SQLException { + deleteExecutor.clearBatch(); + upsertExecutor.clearBatch(); + } + + @Override + public void reOpen(Connection connection) throws SQLException { + deleteExecutor.reOpen(connection); + upsertExecutor.reOpen(connection); + } + + @Override + public void clearStatementCache() {} +} diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/proxy/CachedWrapperProxy.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/proxy/CachedWrapperProxy.java new file mode 100644 index 0000000000..fd7cdb13c8 --- /dev/null +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/proxy/CachedWrapperProxy.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dtstack.chunjun.connector.jdbc.sink.wrapper.proxy; + +import com.dtstack.chunjun.connector.jdbc.sink.wrapper.JdbcBatchStatementWrapper; + +import org.apache.flink.table.data.RowData; + +import com.esotericsoftware.minlog.Log; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +public abstract class CachedWrapperProxy implements JdbcBatchStatementWrapper { + + protected Connection connection; + private final int cacheSize; + private final long cacheDurationMin; + + protected Cache> cache; + + protected JdbcBatchStatementWrapper currentExecutor; + + /** 当调用writeMultipleRecords 可能会涉及到多个executor */ + protected final Set> unExecutedExecutor = + new LinkedHashSet<>(); + + public CachedWrapperProxy( + Connection connection, int cacheSize, long cacheDurationMin, boolean cacheIsExpire) { + this.connection = connection; + this.cacheSize = cacheSize; + this.cacheDurationMin = cacheDurationMin; + initCache(cacheIsExpire); + } + + @Override + public final void addToBatch(RowData record) throws Exception { + record = switchExecutorFromCache(record); + currentExecutor.addToBatch(record); + unExecutedExecutor.add(currentExecutor); + } + + @Override + public final void writeSingleRecord(RowData record) throws Exception { + record = switchExecutorFromCache(record); + currentExecutor.writeSingleRecord(record); + } + + @Override + public final ResultSet executeQuery(RowData record) throws Exception { + record = switchExecutorFromCache(record); + return currentExecutor.executeQuery(record); + } + + @Override + public final void clearParameters() throws SQLException { + currentExecutor.clearParameters(); + } + + @Override + public final void executeBatch() throws Exception { + for (JdbcBatchStatementWrapper executor : unExecutedExecutor) { + executor.executeBatch(); + } + } + + @Override + public final void clearBatch() throws SQLException { + for (JdbcBatchStatementWrapper executor : unExecutedExecutor) { + executor.clearBatch(); + } + unExecutedExecutor.clear(); + } + + @Override + public final void close() throws SQLException { + if (null != currentExecutor) { + currentExecutor.close(); + } + } + + @Override + public final void reOpen(Connection connection) throws SQLException { + this.connection = connection; + ConcurrentMap> + stringDynamicPreparedStmtConcurrentMap = cache.asMap(); + initCache(true); + for (Map.Entry> entry : + stringDynamicPreparedStmtConcurrentMap.entrySet()) { + JdbcBatchStatementWrapper value = entry.getValue(); + value.reOpen(connection); + cache.put(entry.getKey(), value); + currentExecutor = value; + } + } + + protected abstract RowData switchExecutorFromCache(RowData record) throws ExecutionException; + + protected void initCache(boolean isExpired) { + CacheBuilder> cacheBuilder = + CacheBuilder.newBuilder() + .maximumSize(cacheSize) + .removalListener( + notification -> { + try { + assert notification.getValue() != null; + notification.getValue().close(); + } catch (SQLException e) { + Log.error("", e); + } + }); + if (isExpired) { + cacheBuilder.expireAfterAccess(cacheDurationMin, TimeUnit.MINUTES); + } + this.cache = cacheBuilder.build(); + } + + public void clearStatementCache() { + cache.invalidateAll(); + } +} diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/proxy/DynamicSimpleDeleteWrapper.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/proxy/DynamicSimpleDeleteWrapper.java new file mode 100644 index 0000000000..bd84b3ab96 --- /dev/null +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/proxy/DynamicSimpleDeleteWrapper.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dtstack.chunjun.connector.jdbc.sink.wrapper.proxy; + +import com.dtstack.chunjun.connector.jdbc.conf.JdbcConf; +import com.dtstack.chunjun.connector.jdbc.dialect.JdbcDialect; +import com.dtstack.chunjun.connector.jdbc.sink.wrapper.SimpleStatementWrapper; +import com.dtstack.chunjun.connector.jdbc.statement.FieldNamedPreparedStatement; +import com.dtstack.chunjun.connector.jdbc.statement.FieldNamedPreparedStatementImpl; +import com.dtstack.chunjun.converter.AbstractRowConverter; +import com.dtstack.chunjun.util.TableUtil; + +import org.apache.flink.table.types.logical.RowType; + +import java.lang.reflect.Method; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +/** + * base on row data info to build preparedStatement. row data info include rowkind(which is to set + * which sql kind to use ) + */ +public class DynamicSimpleDeleteWrapper extends SimpleStatementWrapper { + + protected List columnNameList = new ArrayList<>(); + + protected List columnTypeList = new ArrayList<>(); + protected List nullColumnNameList = new ArrayList<>(); + + protected JdbcConf jdbcConf; + private JdbcDialect jdbcDialect; + + public DynamicSimpleDeleteWrapper( + FieldNamedPreparedStatement statement, AbstractRowConverter converter) { + super(statement, converter); + } + + public static DynamicSimpleDeleteWrapper buildExecutor( + List columnNameList, + List columnTypeList, + List nullColumnNameList, + String schemaName, + String tableName, + Connection connection, + JdbcDialect jdbcDialect) + throws SQLException { + + DynamicSimpleDeleteWrapper dynamicPreparedStmt = new DynamicSimpleDeleteWrapper(null, null); + + dynamicPreparedStmt.jdbcDialect = jdbcDialect; + dynamicPreparedStmt.columnNameList = columnNameList; + dynamicPreparedStmt.columnTypeList = columnTypeList; + dynamicPreparedStmt.nullColumnNameList = nullColumnNameList; + dynamicPreparedStmt.buildRowConvert(); + + // adapt to pg + dynamicPreparedStmt.adaptToPostgresRowConverter( + dynamicPreparedStmt.rowConverter, connection); + + String sql = dynamicPreparedStmt.prepareTemplates(schemaName, tableName); + String[] fieldNames = new String[dynamicPreparedStmt.columnNameList.size()]; + dynamicPreparedStmt.columnNameList.toArray(fieldNames); + + String[] nullFieldNames = new String[dynamicPreparedStmt.nullColumnNameList.size()]; + dynamicPreparedStmt.nullColumnNameList.toArray(nullFieldNames); + dynamicPreparedStmt.statement = + FieldNamedPreparedStatementImpl.prepareStatement( + connection, sql, fieldNames, nullFieldNames); + return dynamicPreparedStmt; + } + + protected String prepareTemplates(String schemaName, String tableName) { + + String[] columnNames = new String[columnNameList.size()]; + columnNameList.toArray(columnNames); + + String[] nullColumnNames = new String[nullColumnNameList.size()]; + nullColumnNameList.toArray(nullColumnNames); + + return jdbcDialect.getDeleteStatement(schemaName, tableName, columnNames, nullColumnNames); + } + + public void buildRowConvert() { + RowType rowType = + TableUtil.createRowType( + columnNameList, columnTypeList, jdbcDialect.getRawTypeConverter()); + rowConverter = jdbcDialect.getColumnConverter(rowType, jdbcConf); + } + + public void adaptToPostgresRowConverter(AbstractRowConverter rowConv, Connection conn) { + try { + if (rowConv.getClass().getSimpleName().equals("PostgresqlColumnConverter")) { + Class pgConn = Class.forName("org.postgresql.core.BaseConnection"); + Method setConnection = + rowConv.getClass().getDeclaredMethod("setConnection", pgConn); + setConnection.invoke(rowConv, conn); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/proxy/NoKeyDeleteWrapperProxy.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/proxy/NoKeyDeleteWrapperProxy.java new file mode 100644 index 0000000000..fff40c0e4a --- /dev/null +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/proxy/NoKeyDeleteWrapperProxy.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dtstack.chunjun.connector.jdbc.sink.wrapper.proxy; + +import com.dtstack.chunjun.connector.jdbc.dialect.JdbcDialect; +import com.dtstack.chunjun.element.ColumnRowData; +import com.dtstack.chunjun.element.column.NullColumn; + +import org.apache.flink.table.data.RowData; + +import java.sql.Connection; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; + +public class NoKeyDeleteWrapperProxy extends CachedWrapperProxy { + + private final String schema; + private final String table; + private final JdbcDialect jdbcDialect; + private final List columnNameList; + private final List columnTypeList; + + private List nullColumnNameList = new ArrayList<>(); + + public NoKeyDeleteWrapperProxy( + Connection connection, + String schema, + String table, + JdbcDialect jdbcDialect, + List columnNameList, + List columnTypeList) { + super(connection, 100, 10, true); + this.schema = schema; + this.table = table; + this.jdbcDialect = jdbcDialect; + this.columnNameList = columnNameList; + this.columnTypeList = columnTypeList; + } + + @Override + protected RowData switchExecutorFromCache(RowData record) throws ExecutionException { + String cacheKey = getCacheKey((ColumnRowData) record); + currentExecutor = + cache.get( + cacheKey, + () -> + DynamicSimpleDeleteWrapper.buildExecutor( + columnNameList, + columnTypeList, + nullColumnNameList, + schema, + table, + connection, + jdbcDialect)); + return record; + } + + private String getCacheKey(ColumnRowData columnRowData) { + nullColumnNameList.clear(); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < columnNameList.size(); i++) { + if (columnRowData.getField(i) instanceof NullColumn || columnRowData.isNullAt(i)) { + sb.append(1); + nullColumnNameList.add(columnNameList.get(i)); + } else { + sb.append(0); + } + } + return sb.toString(); + } +} diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/proxy/RestoreWrapperProxy.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/proxy/RestoreWrapperProxy.java new file mode 100644 index 0000000000..69d07ed42e --- /dev/null +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/sink/wrapper/proxy/RestoreWrapperProxy.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dtstack.chunjun.connector.jdbc.sink.wrapper.proxy; + +import com.dtstack.chunjun.connector.jdbc.conf.JdbcConf; +import com.dtstack.chunjun.connector.jdbc.dialect.JdbcDialect; +import com.dtstack.chunjun.connector.jdbc.sink.wrapper.StatementWrapperUtil; +import com.dtstack.chunjun.constants.CDCConstantValue; +import com.dtstack.chunjun.element.ColumnRowData; + +import org.apache.flink.table.data.RowData; + +import org.apache.commons.collections.MapUtils; + +import java.sql.Connection; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +/** + * build prepare proxy, proxy implements FieldNamedPreparedStatement. it support to build + * prepareStmt and manager it with cache. + */ +public class RestoreWrapperProxy extends CachedWrapperProxy { + + protected JdbcDialect jdbcDialect; + protected JdbcConf jdbcConf; + + /** 是否将框架额外添加的扩展信息写入到数据库,默认不写入* */ + protected boolean writeExtInfo; + + public RestoreWrapperProxy( + Connection connection, JdbcDialect jdbcDialect, boolean writeExtInfo) { + super(connection, 100, 10, true); + this.jdbcDialect = jdbcDialect; + this.writeExtInfo = writeExtInfo; + } + + @Override + protected RowData switchExecutorFromCache(RowData record) throws ExecutionException { + record = ((ColumnRowData) record).sortColumnRowData(); + ColumnRowData columnRowData = (ColumnRowData) record; + Map head = columnRowData.getHeaderInfo(); + if (MapUtils.isEmpty(head)) { + return record; + } + int dataBaseIndex = head.get(CDCConstantValue.SCHEMA); + int tableIndex = head.get(CDCConstantValue.TABLE); + + String database = record.getString(dataBaseIndex).toString(); + String tableName = record.getString(tableIndex).toString(); + String key = getExecutorCacheKey(database, tableName); + + if (!writeExtInfo) { + columnRowData.removeExtHeaderInfo(); + } + + currentExecutor = + cache.get( + key, + () -> + StatementWrapperUtil.buildInsertDeleteExecutor( + connection, + jdbcDialect, + database, + tableName, + columnRowData)); + return record; + } + + public String getExecutorCacheKey(String schema, String table) { + return String.format("%s_%s", schema, table); + } +} diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/source/JdbcSourceFactory.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/source/JdbcSourceFactory.java index 316519f504..0701732798 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/source/JdbcSourceFactory.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/source/JdbcSourceFactory.java @@ -188,7 +188,7 @@ protected Connection getConn() { protected Pair, List> getTableMetaData(Connection dbConn) { Tuple3 tableIdentify = - jdbcDialect.getTableIdentify().apply(jdbcConf); + jdbcDialect.getTableIdentify(jdbcConf.getSchema(), jdbcConf.getTable()); return JdbcUtil.getTableMetaData( tableIdentify.f0, tableIdentify.f1, diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/statement/FieldNamedPreparedStatement.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/statement/FieldNamedPreparedStatement.java index 2d68871364..86f635d4be 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/statement/FieldNamedPreparedStatement.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/statement/FieldNamedPreparedStatement.java @@ -68,7 +68,8 @@ public interface FieldNamedPreparedStatement extends AutoCloseable { */ static FieldNamedPreparedStatement prepareStatement( Connection connection, String sql, String[] fieldNames) throws SQLException { - return FieldNamedPreparedStatementImpl.prepareStatement(connection, sql, fieldNames); + return FieldNamedPreparedStatementImpl.prepareStatement( + connection, sql, fieldNames, new String[0]); } /** @@ -254,6 +255,8 @@ static FieldNamedPreparedStatement prepareStatement( void setArray(int fieldIndex, Array array) throws SQLException; + boolean isDelete(); + /** * Releases this Statement object's database and JDBC resources immediately instead * of waiting for this to happen when it is automatically closed. It is generally good practice diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/statement/FieldNamedPreparedStatementImpl.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/statement/FieldNamedPreparedStatementImpl.java index d076a3138b..b1236eda3b 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/statement/FieldNamedPreparedStatementImpl.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/statement/FieldNamedPreparedStatementImpl.java @@ -29,8 +29,10 @@ import java.sql.Time; import java.sql.Timestamp; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import static org.apache.flink.util.Preconditions.checkArgument; @@ -43,15 +45,19 @@ public class FieldNamedPreparedStatementImpl implements FieldNamedPreparedStatem private final String parsedSQL; private final int[][] indexMapping; + private final boolean isDelete; + private FieldNamedPreparedStatementImpl( - PreparedStatement statement, String parsedSQL, int[][] indexMapping) { + PreparedStatement statement, String parsedSQL, int[][] indexMapping, boolean isDelete) { this.statement = statement; this.parsedSQL = parsedSQL; this.indexMapping = indexMapping; + this.isDelete = isDelete; } public static FieldNamedPreparedStatement prepareStatement( - Connection connection, String sql, String[] fieldNames) throws SQLException { + Connection connection, String sql, String[] fieldNames, String[] nullFieldNames) + throws SQLException { checkNotNull(connection, "connection must not be null."); checkNotNull(sql, "sql must not be null."); checkNotNull(fieldNames, "fieldNames must not be null."); @@ -60,21 +66,47 @@ public static FieldNamedPreparedStatement prepareStatement( throw new IllegalArgumentException("SQL statement must not contain ? character."); } + boolean isDelete = sql.toLowerCase(Locale.ENGLISH).startsWith("delete"); HashMap> parameterMap = new HashMap<>(); String parsedSQL = parseNamedStatement(sql, parameterMap); - // currently, the statements must contain all the field parameters - checkArgument(parameterMap.size() == fieldNames.length); + // currently, the statements must contain all the field parameters, delete param can skip + // some special type column,example blob clob raw nclob in oracle + if (!isDelete) { + // checkArgument(parameterMap.size() + nullFieldNames.length == + // fieldNames.length); + checkArgument(parameterMap.size() == fieldNames.length); + } + int[][] indexMapping = new int[fieldNames.length][]; + List nullFieldNameList = new ArrayList<>(Arrays.asList(nullFieldNames)); + for (int i = 0; i < fieldNames.length; i++) { String fieldName = fieldNames[i]; - checkArgument( - parameterMap.containsKey(fieldName), - fieldName + " doesn't exist in the parameters of SQL statement: " + sql); - indexMapping[i] = parameterMap.get(fieldName).stream().mapToInt(v -> v).toArray(); + if (isDelete) { + if (nullFieldNameList.contains(fieldName)) { + int[] ints = new int[1]; + ints[0] = i + 1; + indexMapping[i] = ints; + } else { + // delete param can skip some special type column,example blob clob raw nclob in + // oracle + if (parameterMap.containsKey(fieldName)) { + indexMapping[i] = + parameterMap.get(fieldName).stream().mapToInt(v -> v).toArray(); + } else { + indexMapping[i] = null; + } + } + } else { + checkArgument( + parameterMap.containsKey(fieldName), + fieldName + " doesn't exist in the parameters of SQL statement: " + sql); + indexMapping[i] = parameterMap.get(fieldName).stream().mapToInt(v -> v).toArray(); + } } return new FieldNamedPreparedStatementImpl( - connection.prepareStatement(parsedSQL), parsedSQL, indexMapping); + connection.prepareStatement(parsedSQL), parsedSQL, indexMapping, isDelete); } /** @@ -143,6 +175,9 @@ public boolean execute() throws SQLException { @Override public void setNull(int fieldIndex, int sqlType) throws SQLException { + if (isDelete) { + return; + } for (int index : indexMapping[fieldIndex]) { statement.setNull(index, sqlType); } @@ -241,6 +276,9 @@ public void setTimestamp(int fieldIndex, Timestamp x) throws SQLException { @Override public void setObject(int fieldIndex, Object x) throws SQLException { + if (x == null && isDelete) { + return; + } for (int index : indexMapping[fieldIndex]) { statement.setObject(index, x); } @@ -269,6 +307,11 @@ public void setArray(int fieldIndex, Array array) throws SQLException { } } + @Override + public boolean isDelete() { + return isDelete; + } + @Override public void close() throws SQLException { statement.close(); diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/util/JdbcUtil.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/util/JdbcUtil.java index 50f52733ee..1a37fd49f8 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/util/JdbcUtil.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/util/JdbcUtil.java @@ -24,6 +24,7 @@ import com.dtstack.chunjun.connector.jdbc.source.JdbcInputSplit; import com.dtstack.chunjun.constants.ConstantValue; import com.dtstack.chunjun.converter.RawTypeConverter; +import com.dtstack.chunjun.element.ColumnRowData; import com.dtstack.chunjun.throwable.ChunJunRuntimeException; import com.dtstack.chunjun.util.ClassUtil; import com.dtstack.chunjun.util.ExceptionUtil; @@ -32,7 +33,11 @@ import com.dtstack.chunjun.util.TableUtil; import com.dtstack.chunjun.util.TelnetUtil; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.logical.DistinctType; import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -54,8 +59,14 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static org.apache.flink.table.types.logical.utils.LogicalTypeChecks.getFieldCount; +import static org.apache.flink.table.types.logical.utils.LogicalTypeChecks.getPrecision; +import static org.apache.flink.table.types.logical.utils.LogicalTypeChecks.getScale; /** * Utilities for relational database connection and sql execution company: www.dtstack.com @@ -123,6 +134,16 @@ public static Connection getConnection( return getConnection(conf, new JdbcDialectWrapper(dialect)); } + private static String getTableInfo(String catalog, String schema, String tableName) { + if (schema != null) { + return schema + "." + tableName; + } else if (catalog != null) { + return catalog + "." + tableName; + } else { + return tableName; + } + } + /** * get full column name and type from database * @@ -133,15 +154,15 @@ public static Connection getConnection( * @return fullColumnList and fullColumnTypeList */ public static Pair, List> getTableMetaData( - String cataLog, String schema, String tableName, Connection dbConn) { - return getTableMetaData(cataLog, schema, tableName, dbConn, null); + String catalog, String schema, String tableName, Connection dbConn) { + return getTableMetaData(catalog, schema, tableName, dbConn, null); } public static Pair, List> getTableMetaData( - String cataLog, String schema, String tableName, Connection dbConn, String querySql) { + String catalog, String schema, String tableName, Connection dbConn, String querySql) { try { if (StringUtils.isEmpty(schema)) { - schema = cataLog; + schema = catalog; } if (StringUtils.isBlank(querySql)) { // check table exists @@ -149,7 +170,7 @@ public static Pair, List> getTableMetaData( return Pair.of(new LinkedList<>(), new LinkedList<>()); } ResultSet tableRs = - dbConn.getMetaData().getTables(cataLog, schema, tableName, null); + dbConn.getMetaData().getTables(catalog, schema, tableName, null); if (!tableRs.next()) { String tableInfo = schema == null ? tableName : schema + "." + tableName; throw new ChunJunRuntimeException( @@ -643,4 +664,125 @@ public static void setStarLocationForSplits(JdbcInputSplit[] splits, String star } } } + + public static Function getKeyExtractor( + List columnList, + List keyList, + RowType keyRowType, + boolean useAbstractBaseColumn) { + List keyLogicalTypeList = + keyRowType.getFields().stream() + .map(RowType.RowField::getType) + .collect(Collectors.toList()); + int[] keyIndices = new int[keyList.size()]; + for (int i = 0; i < keyList.size(); i++) { + keyIndices[i] = columnList.indexOf(keyList.get(i)); + } + if (useAbstractBaseColumn) { + return row -> { + ColumnRowData pkRow = new ColumnRowData(keyList.size()); + for (int i = 0; i < keyList.size(); i++) { + pkRow.addField(((ColumnRowData) row).getField(keyIndices[i])); + } + return pkRow; + }; + } else { + final RowData.FieldGetter[] fieldGetters = new RowData.FieldGetter[keyList.size()]; + for (int i = 0; i < keyList.size(); i++) { + fieldGetters[i] = createFieldGetter(keyLogicalTypeList.get(i), keyIndices[i]); + } + return row -> { + GenericRowData pkRow = new GenericRowData(fieldGetters.length); + for (int i = 0; i < fieldGetters.length; i++) { + pkRow.setField(i, fieldGetters[i].getFieldOrNull(row)); + } + return pkRow; + }; + } + } + + static RowData.FieldGetter createFieldGetter(LogicalType fieldType, int fieldPos) { + final RowData.FieldGetter fieldGetter; + // ordered by type root definition + switch (fieldType.getTypeRoot()) { + case CHAR: + case VARCHAR: + fieldGetter = row -> row.getString(fieldPos); + break; + case BOOLEAN: + fieldGetter = row -> row.getBoolean(fieldPos); + break; + case BINARY: + case VARBINARY: + fieldGetter = row -> row.getBinary(fieldPos); + break; + case DECIMAL: + final int decimalPrecision = getPrecision(fieldType); + final int decimalScale = getScale(fieldType); + fieldGetter = row -> row.getDecimal(fieldPos, decimalPrecision, decimalScale); + break; + case TINYINT: + fieldGetter = row -> row.getByte(fieldPos); + break; + case SMALLINT: + fieldGetter = row -> row.getShort(fieldPos); + break; + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + case INTERVAL_YEAR_MONTH: + fieldGetter = row -> row.getInt(fieldPos); + break; + case BIGINT: + case INTERVAL_DAY_TIME: + fieldGetter = row -> row.getLong(fieldPos); + break; + case FLOAT: + fieldGetter = row -> row.getFloat(fieldPos); + break; + case DOUBLE: + fieldGetter = row -> row.getDouble(fieldPos); + break; + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + final int timestampPrecision = getPrecision(fieldType); + fieldGetter = row -> row.getTimestamp(fieldPos, timestampPrecision); + break; + case TIMESTAMP_WITH_TIME_ZONE: + throw new UnsupportedOperationException(); + case ARRAY: + fieldGetter = row -> row.getArray(fieldPos); + break; + case MULTISET: + case MAP: + fieldGetter = row -> row.getMap(fieldPos); + break; + case ROW: + case STRUCTURED_TYPE: + final int rowFieldCount = getFieldCount(fieldType); + fieldGetter = row -> row.getRow(fieldPos, rowFieldCount); + break; + case DISTINCT_TYPE: + fieldGetter = + createFieldGetter(((DistinctType) fieldType).getSourceType(), fieldPos); + break; + case RAW: + fieldGetter = row -> row.getRawValue(fieldPos); + break; + case NULL: + case SYMBOL: + case UNRESOLVED: + default: + throw new IllegalArgumentException(); + } + if (!fieldType.isNullable()) { + return fieldGetter; + } + return row -> { + if (row.isNullAt(fieldPos)) { + return null; + } + return fieldGetter.getFieldOrNull(row); + }; + } } diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/test/java/com/dtstack/chunjun/connector/jdbc/dialect/JdbcDialectTest.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/test/java/com/dtstack/chunjun/connector/jdbc/dialect/JdbcDialectTest.java index 3c6b4bda5e..fe7bce4c1a 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/test/java/com/dtstack/chunjun/connector/jdbc/dialect/JdbcDialectTest.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/test/java/com/dtstack/chunjun/connector/jdbc/dialect/JdbcDialectTest.java @@ -39,6 +39,7 @@ public class JdbcDialectTest { private static String schema = "schema"; private static String table = "table"; private static String[] fields = new String[] {"id", "name"}; + private static String[] nullFields = new String[] {}; private static String[] conditionFields = new String[] {"id"}; @BeforeClass @@ -87,9 +88,11 @@ public void getUpdateStatementTest() { @Test public void getDeleteStatementTest() { - when(jdbcDialect.getDeleteStatement(schema, table, fields)).thenCallRealMethod(); + when(jdbcDialect.getDeleteStatement(schema, table, fields, nullFields)) + .thenCallRealMethod(); String expect = "DELETE FROM \"schema\".\"table\" WHERE \"id\" = :id AND \"name\" = :name"; - Assert.assertEquals(expect, jdbcDialect.getDeleteStatement(schema, table, fields)); + Assert.assertEquals( + expect, jdbcDialect.getDeleteStatement(schema, table, fields, nullFields)); } @Test diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/test/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormatTest.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/test/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormatTest.java index 521f691fe0..e623231f78 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/test/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormatTest.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/test/java/com/dtstack/chunjun/connector/jdbc/sink/JdbcOutputFormatTest.java @@ -22,6 +22,7 @@ import com.dtstack.chunjun.connector.jdbc.converter.JdbcColumnConverter; import com.dtstack.chunjun.connector.jdbc.converter.JdbcRawTypeConverterTest; import com.dtstack.chunjun.connector.jdbc.dialect.JdbcDialect; +import com.dtstack.chunjun.connector.jdbc.sink.wrapper.JdbcBatchStatementWrapper; import com.dtstack.chunjun.connector.jdbc.statement.FieldNamedPreparedStatement; import com.dtstack.chunjun.connector.jdbc.util.JdbcUtil; import com.dtstack.chunjun.connector.jdbc.util.SqlUtil; @@ -55,7 +56,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.NoSuchElementException; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; @@ -75,7 +75,7 @@ SqlUtil.class, TableUtil.class, JdbcUtil.class, - PreparedStmtProxy.class, + JdbcBatchStatementWrapper.class, FieldNamedPreparedStatement.class, LongCounter.class }) @@ -88,7 +88,7 @@ public class JdbcOutputFormatTest { BigIntegerAccumulator endLocationAccumulator; FormatState formatState; JdbcColumnConverter rowConverter; - PreparedStmtProxy stmtProxy; + JdbcBatchStatementWrapper statementWrapper; LongCounter snapshotWriteCounter; Connection connection = mock(Connection.class); @@ -111,14 +111,14 @@ public void setup() { endLocationAccumulator = mock(BigIntegerAccumulator.class); formatState = mock(FormatState.class); rowConverter = mock(JdbcColumnConverter.class); - stmtProxy = mock(PreparedStmtProxy.class); + statementWrapper = mock(JdbcBatchStatementWrapper.class); snapshotWriteCounter = mock(LongCounter.class); setInternalState(jdbcOutputFormat, "dbConn", connection); setInternalState(jdbcOutputFormat, "formatState", formatState); setInternalState(jdbcOutputFormat, "jdbcConf", jdbcConf); setInternalState(jdbcOutputFormat, "jdbcDialect", jdbcDialect); - setInternalState(jdbcOutputFormat, "stmtProxy", stmtProxy); + setInternalState(jdbcOutputFormat, "statementWrapper", statementWrapper); setInternalState(jdbcOutputFormat, "snapshotWriteCounter", snapshotWriteCounter); } @@ -142,14 +142,14 @@ public void openInternalTest() throws SQLException { @Test public void buildStmtProxyTest() throws SQLException { - doCallRealMethod().when(jdbcOutputFormat).buildStmtProxy(); + doCallRealMethod().when(jdbcOutputFormat).buildStatementWrapper(); setInternalState(jdbcOutputFormat, "columnNameList", Collections.singletonList("id")); when(jdbcConf.getTable()).thenReturn("*"); - jdbcOutputFormat.buildStmtProxy(); + jdbcOutputFormat.buildStatementWrapper(); when(jdbcConf.getTable()).thenReturn("test_sink"); when(jdbcDialect.getRawTypeConverter()).thenReturn(JdbcRawTypeConverterTest::apply); - jdbcOutputFormat.buildStmtProxy(); + jdbcOutputFormat.buildStatementWrapper(); } /** -------------------------------- write test -------------------------------- */ @@ -160,8 +160,8 @@ public void writeSingleRecordInternalTest() throws Exception { jdbcOutputFormat.writeSingleRecordInternal(new ColumnRowData(1)); doThrow(new SQLException("No operations allowed")) - .when(stmtProxy) - .writeSingleRecordInternal(any(RowData.class)); + .when(statementWrapper) + .writeSingleRecord(any(RowData.class)); doCallRealMethod() .when(jdbcOutputFormat) .processWriteException(any(), anyInt(), any(RowData.class)); @@ -199,7 +199,7 @@ public void writeMultipleRecordsInternalTest() throws Exception { setInternalState(jdbcOutputFormat, "semantic", Semantic.EXACTLY_ONCE); jdbcOutputFormat.writeMultipleRecordsInternal(); - when(stmtProxy.executeBatch()).thenThrow(new SQLException()); + doThrow(new SQLException()).when(statementWrapper).executeBatch(); Assert.assertThrows( SQLException.class, () -> jdbcOutputFormat.writeMultipleRecordsInternal()); } @@ -222,7 +222,7 @@ public void commitTest() throws Exception { setInternalState(jdbcOutputFormat, "rowsOfCurrentTransaction", 1); jdbcOutputFormat.commit(1); - doThrow(new ChunJunRuntimeException("")).when(stmtProxy).clearStatementCache(); + doThrow(new ChunJunRuntimeException("")).when(statementWrapper).clearStatementCache(); Assert.assertThrows(Exception.class, () -> jdbcOutputFormat.commit(1)); } @@ -256,20 +256,23 @@ public void prepareTemplatesTest() { // insert when(jdbcConf.getMode()).thenReturn(EWriteMode.INSERT.name()); when(jdbcDialect.getInsertIntoStatement(any(), any(), any())).thenCallRealMethod(); + when(jdbcOutputFormat.getInsertPrepareTemplate()).thenCallRealMethod(); expect = "INSERT INTO null(null, null) VALUES (:id, :name)"; Assert.assertEquals(expect, jdbcOutputFormat.prepareTemplates()); // replace when(jdbcConf.getMode()).thenReturn(EWriteMode.REPLACE.name()); when(jdbcDialect.getReplaceStatement(any(), any(), any())).thenCallRealMethod(); + when(jdbcOutputFormat.getReplacePrepareTemplate()).thenCallRealMethod(); Assert.assertThrows( - NoSuchElementException.class, () -> jdbcOutputFormat.prepareTemplates()); + IllegalArgumentException.class, () -> jdbcOutputFormat.prepareTemplates()); // update when(jdbcConf.getMode()).thenReturn(EWriteMode.UPDATE.name()); when(jdbcDialect.getUpdateStatement(any(), any(), any(), any())).thenCallRealMethod(); when(jdbcConf.getUniqueKey()).thenReturn(Collections.singletonList("id")); when(jdbcConf.isAllReplace()).thenReturn(true); + when(jdbcOutputFormat.getUpsertStatement()).thenCallRealMethod(); Assert.assertThrows( - NoSuchElementException.class, () -> jdbcOutputFormat.prepareTemplates()); + IllegalArgumentException.class, () -> jdbcOutputFormat.prepareTemplates()); // exception when(jdbcConf.getMode()).thenReturn("asd"); Assert.assertThrows( @@ -282,7 +285,7 @@ public void closeInternalTest() throws SQLException { setInternalState(jdbcOutputFormat, "rowsOfCurrentTransaction", 1); jdbcOutputFormat.closeInternal(); - doThrow(new SQLException("")).when(stmtProxy).close(); + doThrow(new SQLException("")).when(statementWrapper).close(); jdbcOutputFormat.closeInternal(); } } diff --git a/chunjun-connectors/chunjun-connector-mysql/src/main/java/com/dtstack/chunjun/connector/mysql/dialect/MysqlDialect.java b/chunjun-connectors/chunjun-connector-mysql/src/main/java/com/dtstack/chunjun/connector/mysql/dialect/MysqlDialect.java index 724b80e4cd..60fa0b26a8 100644 --- a/chunjun-connectors/chunjun-connector-mysql/src/main/java/com/dtstack/chunjun/connector/mysql/dialect/MysqlDialect.java +++ b/chunjun-connectors/chunjun-connector-mysql/src/main/java/com/dtstack/chunjun/connector/mysql/dialect/MysqlDialect.java @@ -18,7 +18,6 @@ package com.dtstack.chunjun.connector.mysql.dialect; -import com.dtstack.chunjun.connector.jdbc.conf.JdbcConf; import com.dtstack.chunjun.connector.jdbc.dialect.JdbcDialect; import com.dtstack.chunjun.connector.mysql.converter.MysqlRawTypeConverter; import com.dtstack.chunjun.converter.RawTypeConverter; @@ -27,7 +26,6 @@ import java.util.Arrays; import java.util.Optional; -import java.util.function.Function; import java.util.stream.Collectors; /** @@ -62,6 +60,11 @@ public String quoteIdentifier(String identifier) { return "`" + identifier + "`"; } + @Override + public boolean supportUpsert() { + return true; + } + /** * Mysql upsert query use DUPLICATE KEY UPDATE. * @@ -123,7 +126,7 @@ public Optional getReplaceStatement( } @Override - public Function> getTableIdentify() { - return conf -> Tuple3.of(conf.getSchema(), null, conf.getTable()); + public Tuple3 getTableIdentify(String confSchema, String confTable) { + return Tuple3.of(confSchema, null, confTable); } } diff --git a/chunjun-connectors/chunjun-connector-oracle/src/main/java/com/dtstack/chunjun/connector/oracle/dialect/OracleDialect.java b/chunjun-connectors/chunjun-connector-oracle/src/main/java/com/dtstack/chunjun/connector/oracle/dialect/OracleDialect.java index 9569b2b9df..26370ecfca 100644 --- a/chunjun-connectors/chunjun-connector-oracle/src/main/java/com/dtstack/chunjun/connector/oracle/dialect/OracleDialect.java +++ b/chunjun-connectors/chunjun-connector-oracle/src/main/java/com/dtstack/chunjun/connector/oracle/dialect/OracleDialect.java @@ -78,6 +78,11 @@ public Optional getReplaceStatement( throw new RuntimeException("Oracle does not support replace sql"); } + @Override + public boolean supportUpsert() { + return true; + } + @Override public Optional getUpsertStatement( String schema, diff --git a/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/dialect/PostgresqlDialect.java b/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/dialect/PostgresqlDialect.java index 87c3f04125..62acb6e4b4 100644 --- a/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/dialect/PostgresqlDialect.java +++ b/chunjun-connectors/chunjun-connector-postgresql/src/main/java/com/dtstack/chunjun/connector/postgresql/dialect/PostgresqlDialect.java @@ -78,6 +78,11 @@ public Optional defaultDriverName() { return Optional.of(DRIVER); } + @Override + public boolean supportUpsert() { + return true; + } + /** Postgres upsert query. It use ON CONFLICT ... DO UPDATE SET.. to replace into Postgres. */ @Override public Optional getUpsertStatement( diff --git a/chunjun-connectors/chunjun-connector-saphana/src/main/java/com/dtstack/chunjun/connector/saphana/dialect/SaphanaDialect.java b/chunjun-connectors/chunjun-connector-saphana/src/main/java/com/dtstack/chunjun/connector/saphana/dialect/SaphanaDialect.java index afad5bee6c..2d29dd331e 100644 --- a/chunjun-connectors/chunjun-connector-saphana/src/main/java/com/dtstack/chunjun/connector/saphana/dialect/SaphanaDialect.java +++ b/chunjun-connectors/chunjun-connector-saphana/src/main/java/com/dtstack/chunjun/connector/saphana/dialect/SaphanaDialect.java @@ -66,6 +66,11 @@ public Optional defaultDriverName() { return Optional.of("com.sap.db.jdbc.Driver"); } + @Override + public boolean supportUpsert() { + return true; + } + @Override public Optional getUpsertStatement( String schema, diff --git a/chunjun-connectors/chunjun-connector-sqlserver/src/main/java/com/dtstack/chunjun/connector/sqlserver/dialect/SqlserverDialect.java b/chunjun-connectors/chunjun-connector-sqlserver/src/main/java/com/dtstack/chunjun/connector/sqlserver/dialect/SqlserverDialect.java index 9d1cff4c67..46cc5393b9 100644 --- a/chunjun-connectors/chunjun-connector-sqlserver/src/main/java/com/dtstack/chunjun/connector/sqlserver/dialect/SqlserverDialect.java +++ b/chunjun-connectors/chunjun-connector-sqlserver/src/main/java/com/dtstack/chunjun/connector/sqlserver/dialect/SqlserverDialect.java @@ -173,6 +173,11 @@ public String getSplitModFilter(JdbcInputSplit split, String splitPkName) { quoteIdentifier(splitPkName), split.getTotalNumberOfSplits(), split.getMod()); } + @Override + public boolean supportUpsert() { + return true; + } + @Override public Optional getUpsertStatement( String schema, diff --git a/chunjun-connectors/chunjun-connector-sybase/src/main/java/com/dtstack/chunjun/connector/sybase/converter/SybaseColumnConverter.java b/chunjun-connectors/chunjun-connector-sybase/src/main/java/com/dtstack/chunjun/connector/sybase/converter/SybaseColumnConverter.java index 91c324e267..62398c67ad 100644 --- a/chunjun-connectors/chunjun-connector-sybase/src/main/java/com/dtstack/chunjun/connector/sybase/converter/SybaseColumnConverter.java +++ b/chunjun-connectors/chunjun-connector-sybase/src/main/java/com/dtstack/chunjun/connector/sybase/converter/SybaseColumnConverter.java @@ -60,7 +60,13 @@ protected IDeserializationConverter createInternalConverter(LogicalType type) { } }; case TINYINT: - return val -> new BigDecimalColumn(((Integer) val).byteValue()); + return val -> { + if (val instanceof Boolean) { + return new BigDecimalColumn(((boolean) val) ? 1 : 0); + } else { + return new BigDecimalColumn((Integer) val); + } + }; case SMALLINT: case INTEGER: return val -> new BigDecimalColumn((Integer) val); diff --git a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/sink/Vertica11OutputFormat.java b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/sink/Vertica11OutputFormat.java index 004164aafe..99f9019aa8 100644 --- a/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/sink/Vertica11OutputFormat.java +++ b/chunjun-connectors/chunjun-connector-vertica11/src/main/java/com/dtstack/chunjun/connector/vertica11/sink/Vertica11OutputFormat.java @@ -50,7 +50,7 @@ protected void openInternal(int taskNumber, int numTasks) { } } - buildStmtProxy(); + buildStatementWrapper(); LOG.info("subTask[{}}] wait finished", taskNumber); } catch (SQLException sqe) { throw new IllegalArgumentException("open() failed.", sqe); @@ -60,29 +60,15 @@ protected void openInternal(int taskNumber, int numTasks) { } @Override - protected String prepareTemplates() { - String singleSql; - if (EWriteMode.INSERT.name().equalsIgnoreCase(jdbcConf.getMode())) { - singleSql = - jdbcDialect.getInsertIntoStatement( - jdbcConf.getSchema(), - jdbcConf.getTable(), - columnNameList.toArray(new String[0])); - } else if (EWriteMode.UPDATE.name().equalsIgnoreCase(jdbcConf.getMode())) { - singleSql = - ((Vertica11Dialect) jdbcDialect) - .getUpsertStatement( - jdbcConf.getSchema(), - jdbcConf.getTable(), - columnNameList.toArray(new String[0]), - columnTypeList.toArray(new String[0]), - jdbcConf.getUniqueKey().toArray(new String[0]), - jdbcConf.isAllReplace()) - .get(); - } else { - throw new IllegalArgumentException("Unknown write mode:" + jdbcConf.getMode()); - } - LOG.info("write sql:{}", singleSql); - return singleSql; + protected String getUpsertStatement() { + return ((Vertica11Dialect) jdbcDialect) + .getUpsertStatement( + jdbcConf.getSchema(), + jdbcConf.getTable(), + columnNameList.toArray(new String[0]), + columnTypeList.toArray(new String[0]), + jdbcConf.getUniqueKey().toArray(new String[0]), + jdbcConf.isAllReplace()) + .get(); } } diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/element/ColumnRowData.java b/chunjun-core/src/main/java/com/dtstack/chunjun/element/ColumnRowData.java index d27c736f09..d303ffcbf4 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/element/ColumnRowData.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/element/ColumnRowData.java @@ -37,6 +37,7 @@ import java.io.Serializable; import java.math.BigDecimal; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -318,4 +319,70 @@ private String buildString(StringBuilder sb) { sb.append(")"); return sb.toString(); } + + // sort rowData by field header name + public ColumnRowData sortColumnRowData() { + String[] oldHeaders = this.getHeaders(); + + // mode: sync + if (oldHeaders == null) { + return this; + } + + ColumnRowData newRowData = new ColumnRowData(this.getArity()); + String[] newHeaders = Arrays.stream(oldHeaders).sorted().toArray(String[]::new); + + newRowData.setRowKind(this.getRowKind()); + Arrays.stream(newHeaders) + .forEach( + header -> { + AbstractBaseColumn column = this.getField(header); + newRowData.addHeader(header); + newRowData.addField(column); + }); + + Set extHeaders = this.getExtHeader(); + extHeaders.forEach(extHeader -> newRowData.addExtHeader(extHeader)); + return newRowData; + } + + @Override + public int hashCode() { + if (columnList == null) { + return 0; + } + int result = 1; + for (AbstractBaseColumn column : columnList) { + result = 31 * result + (column.data == null ? 0 : column.data.hashCode()); + } + return result; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o instanceof ColumnRowData) { + ColumnRowData that = (ColumnRowData) o; + if (this.columnList.size() != that.columnList.size()) { + return false; + } + Object thisData; + Object thatData; + for (int i = 0; i < this.columnList.size(); i++) { + thisData = this.columnList.get(i).data; + thatData = that.columnList.get(i).data; + if (thisData == null) { + if (thatData != null) { + return false; + } + } else if (!thisData.equals(thatData)) { + return false; + } + } + return true; + } + return false; + } } diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/sink/format/BaseRichOutputFormat.java b/chunjun-core/src/main/java/com/dtstack/chunjun/sink/format/BaseRichOutputFormat.java index b9ea5a2639..cce5fb69fe 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/sink/format/BaseRichOutputFormat.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/sink/format/BaseRichOutputFormat.java @@ -185,7 +185,7 @@ public abstract class BaseRichOutputFormat extends RichOutputFormat @Override public void initializeGlobal(int parallelism) { - // 任务开始前操作，在configure前调用。 + // 任务开始前操作，在configure后调用。 } @Override diff --git a/chunjun-examples/sql/starrocks/stream_starrocks.sql b/chunjun-examples/sql/starrocks/stream_starrocks.sql index 580e336520..915d24ef39 100644 --- a/chunjun-examples/sql/starrocks/stream_starrocks.sql +++ b/chunjun-examples/sql/starrocks/stream_starrocks.sql @@ -16,7 +16,7 @@ CREATE TABLE sink ) with ( 'connector' = 'starrocks-x', 'url' = 'jdbc:mysql://node1:9030', - 'feNodes' = 'node1:8030;node2:8030;node3:8030', + 'fe-nodes' = 'node1:8030;node2:8030;node3:8030', 'schema-name' = 'test', 'table-name' = 'sink', 'username' = 'root', From 3015a84e17dfef47a77f466473064728212756a5 Mon Sep 17 00:00:00 2001 From: xianxing Date: Mon, 11 Dec 2023 09:55:09 +0800 Subject: [PATCH 37/41] [bugfix][hbase] fix deliver a null string causes dimensional table timeout problem --- .../table/lookup/HBaseLruTableFunction.java | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/table/lookup/HBaseLruTableFunction.java b/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/table/lookup/HBaseLruTableFunction.java index a8babf6a4b..088cf3edb0 100644 --- a/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/table/lookup/HBaseLruTableFunction.java +++ b/chunjun-connectors/chunjun-connector-hbase-base/src/main/java/com/dtstack/chunjun/connector/hbase/table/lookup/HBaseLruTableFunction.java @@ -107,28 +107,31 @@ public void run() { byte[] key = serde.getRowKey(rowKey); String keyStr = new String(key); try { - Get get = new Get(key); - Result result = table.get(get); - if (!result.isEmpty()) { - RowData data = serde.convertToNewRow(result); - if (openCache()) { - sideCache.putCache( - keyStr, - CacheObj.buildCacheObj( - ECacheContentType.MultiLine, - Collections.singletonList(data))); - } - future.complete(Collections.singletonList(data)); - } else { - dealMissKey(future); - if (openCache()) { - sideCache.putCache(keyStr, CacheMissVal.getMissKeyObj()); + if (!keyStr.isEmpty()) { + Get get = new Get(key); + Result result = table.get(get); + if (!result.isEmpty()) { + RowData data = serde.convertToNewRow(result); + if (openCache()) { + sideCache.putCache( + keyStr, + CacheObj.buildCacheObj( + ECacheContentType.MultiLine, + Collections.singletonList(data))); + } + future.complete(Collections.singletonList(data)); + return; } } + dealMissKey(future); + if (openCache()) { + sideCache.putCache(keyStr, CacheMissVal.getMissKeyObj()); + } + } catch (IOException e) { LOG.error("record:" + keyStr); LOG.error("get side record exception:" + e); - future.complete(Collections.emptyList()); + dealMissKey(future); } } }); From efbf119cff1344337a048e71920d7e1d87525fad Mon Sep 17 00:00:00 2001 From: xianxing Date: Mon, 11 Dec 2023 10:10:16 +0800 Subject: [PATCH 38/41] [bugfix][redis] fix application failed when redis single node down --- chunjun-connectors/chunjun-connector-redis/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chunjun-connectors/chunjun-connector-redis/pom.xml b/chunjun-connectors/chunjun-connector-redis/pom.xml index c6238e0dc3..d4d4f4fbb0 100644 --- a/chunjun-connectors/chunjun-connector-redis/pom.xml +++ b/chunjun-connectors/chunjun-connector-redis/pom.xml @@ -35,7 +35,7 @@ redis.clients jedis - 2.9.0 + 2.10.1 io.lettuce From 6ef21b6822b1db747b6129c776ad24f63842d3d9 Mon Sep 17 00:00:00 2001 From: xianxing Date: Mon, 11 Dec 2023 10:17:39 +0800 Subject: [PATCH 39/41] [bugfix][kudu] fix postgresql dimension table appear class conflict error when kudu as sink table --- chunjun-connectors/chunjun-connector-kudu/pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/chunjun-connectors/chunjun-connector-kudu/pom.xml b/chunjun-connectors/chunjun-connector-kudu/pom.xml index 6352211e35..9d5711e64a 100644 --- a/chunjun-connectors/chunjun-connector-kudu/pom.xml +++ b/chunjun-connectors/chunjun-connector-kudu/pom.xml @@ -69,6 +69,10 @@ + + io.netty + shade.kudu.io.netty + com.google.common shade.core.com.google.common From 47a72c712c16549f5b613a2ca147e575eb60d4eb Mon Sep 17 00:00:00 2001 From: dou-dou <2313776747@qq.com> Date: Tue, 4 Nov 2025 19:36:14 +0800 Subject: [PATCH 40/41] [bugfix][iceberg] support long type (#1887) Co-authored-by: doushuming --- .../chunjun/connector/iceberg/sink/ChunjunRowDataConvertMap.java | 1 + 1 file changed, 1 insertion(+) diff --git a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/sink/ChunjunRowDataConvertMap.java b/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/sink/ChunjunRowDataConvertMap.java index e6e7c344c9..6dd3ef9179 100644 --- a/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/sink/ChunjunRowDataConvertMap.java +++ b/chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/sink/ChunjunRowDataConvertMap.java @@ -60,6 +60,7 @@ private Object getRowDataByType(RowData data, String type, int index) { case "INT24": case "INT UNSIGNED": return data.getInt(index); + case "LONG": case "BIGINT": case "BIGINT UNSIGNED": return data.getLong(index); From 8422832950789f8ed68970d2ddc1b3172266065e Mon Sep 17 00:00:00 2001 From: kino <56473938+kinoxyz1@users.noreply.github.com> Date: Fri, 21 Nov 2025 18:49:58 +0800 Subject: [PATCH 41/41] [Feature][jdbc splitpk] support manually set the Min/Max value (#1541) --- .../chunjun/connector/jdbc/conf/JdbcConf.java | 24 +++++++++++ .../jdbc/source/JdbcInputFormat.java | 40 +++++++++++-------- .../oracle/oracle-source.md" | 26 ++++++++++++ 3 files changed, 74 insertions(+), 16 deletions(-) diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/conf/JdbcConf.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/conf/JdbcConf.java index 6156284152..5035fb9588 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/conf/JdbcConf.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/conf/JdbcConf.java @@ -56,6 +56,8 @@ public class JdbcConf extends ChunJunCommonConf implements Serializable { protected String orderByColumn; protected String querySql; protected String splitPk; + protected String splitPkStart; + protected String splitPkEnd; protected String splitStrategy; protected int fetchSize = 0; protected int queryTimeOut = 0; @@ -142,6 +144,22 @@ public String getTable() { return connection.get(0).getTable().get(0); } + public String getSplitPkStart() { + return splitPkStart; + } + + public void setSplitPkStart(String splitPkStart) { + this.splitPkStart = splitPkStart; + } + + public String getSplitPkEnd() { + return splitPkEnd; + } + + public void setSplitPkEnd(String splitPkEnd) { + this.splitPkEnd = splitPkEnd; + } + public void setTable(String table) { connection.get(0).getTable().set(0, table); } @@ -485,6 +503,12 @@ public String toString() { + ", splitPk='" + splitPk + '\'' + + ", splitPkStart='" + + splitPkStart + + '\'' + + ", splitPkEnd='" + + splitPkEnd + + '\'' + ", splitStrategy='" + splitStrategy + '\'' diff --git a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/source/JdbcInputFormat.java b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/source/JdbcInputFormat.java index 1690444439..8feb3b560b 100644 --- a/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/source/JdbcInputFormat.java +++ b/chunjun-connectors/chunjun-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/source/JdbcInputFormat.java @@ -394,25 +394,32 @@ private Pair getSplitRangeFromDb() { try { long startTime = System.currentTimeMillis(); - String querySplitRangeSql = SqlUtil.buildQuerySplitRangeSql(jdbcConf, jdbcDialect); - LOG.info(String.format("Query SplitRange sql is '%s'", querySplitRangeSql)); - - conn = getConnection(); - st = conn.createStatement(resultSetType, resultSetConcurrency); - st.setQueryTimeout(jdbcConf.getQueryTimeOut()); - rs = st.executeQuery(querySplitRangeSql); - if (rs.next()) { + if (jdbcConf.getSplitPkStart() != null && jdbcConf.getSplitPkEnd() != null) { splitPkRange = Pair.of( - String.valueOf(rs.getObject("min_value")), - String.valueOf(rs.getObject("max_value"))); - } + String.valueOf(jdbcConf.getSplitPkStart()), + String.valueOf(jdbcConf.getSplitPkEnd())); - LOG.info( - String.format( - "Takes [%s] milliseconds to get the SplitRange value [%s]", - System.currentTimeMillis() - startTime, splitPkRange)); + } else { + String querySplitRangeSql = SqlUtil.buildQuerySplitRangeSql(jdbcConf, jdbcDialect); + LOG.info(String.format("Query SplitRange sql is '%s'", querySplitRangeSql)); + + conn = getConnection(); + st = conn.createStatement(resultSetType, resultSetConcurrency); + st.setQueryTimeout(jdbcConf.getQueryTimeOut()); + rs = st.executeQuery(querySplitRangeSql); + if (rs.next()) { + splitPkRange = + Pair.of( + String.valueOf(rs.getObject("min_value")), + String.valueOf(rs.getObject("max_value"))); + } + LOG.info( + String.format( + "Takes [%s] milliseconds to get the SplitRange value [%s]", + System.currentTimeMillis() - startTime, splitPkRange)); + } return splitPkRange; } catch (Throwable e) { throw new ChunJunRuntimeException( @@ -806,7 +813,8 @@ protected Connection getConnection() throws SQLException { /** 使用自定义的指标输出器把增量指标打到普罗米修斯 */ @Override protected boolean useCustomReporter() { - return jdbcConf.isIncrement() && jdbcConf.getInitReporter(); + // 配置了 reporter 就可以输入指标到外部系统, 如果不是增量, 增量指标也不会被输出 + return jdbcConf.getInitReporter(); } /** 为了保证增量数据的准确性，指标输出失败时使任务失败 */ diff --git "a/docs_zh/ChunJun\350\277\236\346\216\245\345\231\250/oracle/oracle-source.md" "b/docs_zh/ChunJun\350\277\236\346\216\245\345\231\250/oracle/oracle-source.md" index 735b793294..b19b0c6cc6 100644 --- "a/docs_zh/ChunJun\350\277\236\346\216\245\345\231\250/oracle/oracle-source.md" +++ "b/docs_zh/ChunJun\350\277\236\346\216\245\345\231\250/oracle/oracle-source.md" @@ -110,6 +110,32 @@ Oracle 9 及以上 - 默认值：无
+- **splitPkStart** + + - 描述: 当指定了 `splitPk` 之后，可以手动指定 `splitPk` 的上/下界，之后可以直接根据这两个值计算每个并行度的数据量。 + - 注意： + - 此参数生效的前提是设置了 `splitPk`，并且需要同时设置 **splitPkEnd** 参数。 + - 推荐splitPk使用表主键，因为表主键通常情况下比较均匀，因此切分出来的分片也不容易出现数据热点。 + - 目前splitPk仅支持整形数据切分，不支持浮点、字符串、日期等其他类型。如果用户指定其他非支持类型，ChunJun将报错。 + - 如果channel大于1但是没有配置此参数，任务将置为失败。 + - 必选：否 + - 参数类型：String + - 默认值：无 +
+ +- **splitPkEnd** + + - 描述: 当指定了 `splitPk` 之后，可以手动指定 `splitPk` 的上/下界，之后可以直接根据这两个值计算每个并行度的数据量。 + - 注意： + - 此参数生效的前提是设置了 `splitPk`，并且需要同时设置 **splitPkStart** 参数。 + - 推荐splitPk使用表主键，因为表主键通常情况下比较均匀，因此切分出来的分片也不容易出现数据热点。 + - 目前splitPk仅支持整形数据切分，不支持浮点、字符串、日期等其他类型。如果用户指定其他非支持类型，ChunJun将报错。 + - 如果channel大于1但是没有配置此参数，任务将置为失败。 + - 必选：否 + - 参数类型：String + - 默认值：无 +
+ - **queryTimeOut** - 描述：查询超时时间，单位秒。