From 3abc33074b0ed49a46817c2572a471620738b925 Mon Sep 17 00:00:00 2001 From: Adam Leszczynski Date: Tue, 28 Jul 2026 19:29:42 +0200 Subject: [PATCH] dev version of oracle to Debezium tutorials --- README.md | 15 ++- tutorials/Oracle-to-Debezium-both/1.check.sh | 27 ++++ .../Oracle-to-Debezium-both/10.db-drop.sh | 28 ++++ .../Oracle-to-Debezium-both/2.db-start.sh | 30 +++++ .../Oracle-to-Debezium-both/3.kafka-start.sh | 31 +++++ .../Oracle-to-Debezium-both/4.olr-start.sh | 30 +++++ .../5.debezium-start.sh | 33 +++++ tutorials/Oracle-to-Debezium-both/6.test.sh | 28 ++++ .../7.debezium-drop.sh | 30 +++++ .../Oracle-to-Debezium-both/8.olr-drop.sh | 28 ++++ .../Oracle-to-Debezium-both/9.kafka-drop.sh | 27 ++++ tutorials/Oracle-to-Debezium-both/README.md | 116 +++++++++++++++++ tutorials/Oracle-to-Debezium-both/cfg.sh | 46 +++++++ .../Oracle-to-Debezium-both/compose.yaml | 120 ++++++++++++++++++ .../scripts/OpenLogReplicator.json | 43 +++++++ .../Oracle-to-Debezium-both/setup/config.sql | 10 ++ .../sql/drop-usrdbz_logminer.sql | 7 + .../sql/drop-usrdbz_olr.sql | 7 + .../sql/drop-usrolr.sql | 1 + .../sql/schema-usrdbz_logminer.sql | 34 +++++ .../sql/schema-usrdbz_olr.sql | 30 +++++ .../sql/schema-usrolr.sql | 46 +++++++ .../sql/schema-usrtbl.sql | 27 ++++ .../Oracle-to-Debezium-both/sql/test.sql | 12 ++ .../Oracle-to-Debezium-logminer/1.check.sh | 26 ++++ .../Oracle-to-Debezium-logminer/2.db-start.sh | 30 +++++ .../3.kafka-start.sh | 31 +++++ .../4.debezium-start.sh | 29 +++++ .../Oracle-to-Debezium-logminer/5.test.sh | 44 +++++++ .../6.debezium-drop.sh | 28 ++++ .../7.kafka-drop.sh | 27 ++++ .../Oracle-to-Debezium-logminer/8.db-drop.sh | 28 ++++ .../Oracle-to-Debezium-logminer/README.md | 117 +++++++++++++++++ tutorials/Oracle-to-Debezium-logminer/cfg.sh | 41 ++++++ .../Oracle-to-Debezium-logminer/compose.yaml | 101 +++++++++++++++ .../setup/config.sql | 10 ++ .../sql/drop-usrdbz_logminer.sql | 7 + .../sql/schema-usrdbz_logminer.sql | 34 +++++ .../sql/schema-usrtbl.sql | 24 ++++ .../Oracle-to-Debezium-logminer/sql/test.sql | 6 + tutorials/Oracle-to-Debezium-olr/1.check.sh | 27 ++++ .../Oracle-to-Debezium-olr/10.db-drop.sh | 28 ++++ .../Oracle-to-Debezium-olr/2.db-start.sh | 30 +++++ .../Oracle-to-Debezium-olr/3.kafka-start.sh | 31 +++++ .../Oracle-to-Debezium-olr/4.olr-start.sh | 30 +++++ .../5.debezium-start.sh | 29 +++++ tutorials/Oracle-to-Debezium-olr/6.test.sh | 50 ++++++++ .../Oracle-to-Debezium-olr/7.debezium-drop.sh | 28 ++++ .../Oracle-to-Debezium-olr/8.olr-drop.sh | 28 ++++ .../Oracle-to-Debezium-olr/9.kafka-drop.sh | 27 ++++ tutorials/Oracle-to-Debezium-olr/README.md | 116 +++++++++++++++++ tutorials/Oracle-to-Debezium-olr/cfg.sh | 42 ++++++ tutorials/Oracle-to-Debezium-olr/compose.yaml | 120 ++++++++++++++++++ .../scripts/OpenLogReplicator.json | 43 +++++++ .../Oracle-to-Debezium-olr/setup/config.sql | 10 ++ .../sql/drop-usrdbz_olr.sql | 7 + .../sql/drop-usrolr.sql | 1 + .../sql/schema-usrdbz_olr.sql | 30 +++++ .../sql/schema-usrolr.sql | 46 +++++++ .../sql/schema-usrtbl.sql | 24 ++++ tutorials/Oracle-to-Debezium-olr/sql/test.sql | 6 + 61 files changed, 2139 insertions(+), 3 deletions(-) create mode 100755 tutorials/Oracle-to-Debezium-both/1.check.sh create mode 100755 tutorials/Oracle-to-Debezium-both/10.db-drop.sh create mode 100755 tutorials/Oracle-to-Debezium-both/2.db-start.sh create mode 100755 tutorials/Oracle-to-Debezium-both/3.kafka-start.sh create mode 100755 tutorials/Oracle-to-Debezium-both/4.olr-start.sh create mode 100755 tutorials/Oracle-to-Debezium-both/5.debezium-start.sh create mode 100755 tutorials/Oracle-to-Debezium-both/6.test.sh create mode 100755 tutorials/Oracle-to-Debezium-both/7.debezium-drop.sh create mode 100755 tutorials/Oracle-to-Debezium-both/8.olr-drop.sh create mode 100755 tutorials/Oracle-to-Debezium-both/9.kafka-drop.sh create mode 100644 tutorials/Oracle-to-Debezium-both/README.md create mode 100755 tutorials/Oracle-to-Debezium-both/cfg.sh create mode 100644 tutorials/Oracle-to-Debezium-both/compose.yaml create mode 100644 tutorials/Oracle-to-Debezium-both/scripts/OpenLogReplicator.json create mode 100644 tutorials/Oracle-to-Debezium-both/setup/config.sql create mode 100644 tutorials/Oracle-to-Debezium-both/sql/drop-usrdbz_logminer.sql create mode 100644 tutorials/Oracle-to-Debezium-both/sql/drop-usrdbz_olr.sql create mode 100644 tutorials/Oracle-to-Debezium-both/sql/drop-usrolr.sql create mode 100644 tutorials/Oracle-to-Debezium-both/sql/schema-usrdbz_logminer.sql create mode 100644 tutorials/Oracle-to-Debezium-both/sql/schema-usrdbz_olr.sql create mode 100644 tutorials/Oracle-to-Debezium-both/sql/schema-usrolr.sql create mode 100644 tutorials/Oracle-to-Debezium-both/sql/schema-usrtbl.sql create mode 100644 tutorials/Oracle-to-Debezium-both/sql/test.sql create mode 100755 tutorials/Oracle-to-Debezium-logminer/1.check.sh create mode 100755 tutorials/Oracle-to-Debezium-logminer/2.db-start.sh create mode 100755 tutorials/Oracle-to-Debezium-logminer/3.kafka-start.sh create mode 100755 tutorials/Oracle-to-Debezium-logminer/4.debezium-start.sh create mode 100755 tutorials/Oracle-to-Debezium-logminer/5.test.sh create mode 100755 tutorials/Oracle-to-Debezium-logminer/6.debezium-drop.sh create mode 100755 tutorials/Oracle-to-Debezium-logminer/7.kafka-drop.sh create mode 100755 tutorials/Oracle-to-Debezium-logminer/8.db-drop.sh create mode 100644 tutorials/Oracle-to-Debezium-logminer/README.md create mode 100755 tutorials/Oracle-to-Debezium-logminer/cfg.sh create mode 100644 tutorials/Oracle-to-Debezium-logminer/compose.yaml create mode 100644 tutorials/Oracle-to-Debezium-logminer/setup/config.sql create mode 100644 tutorials/Oracle-to-Debezium-logminer/sql/drop-usrdbz_logminer.sql create mode 100644 tutorials/Oracle-to-Debezium-logminer/sql/schema-usrdbz_logminer.sql create mode 100644 tutorials/Oracle-to-Debezium-logminer/sql/schema-usrtbl.sql create mode 100644 tutorials/Oracle-to-Debezium-logminer/sql/test.sql create mode 100755 tutorials/Oracle-to-Debezium-olr/1.check.sh create mode 100755 tutorials/Oracle-to-Debezium-olr/10.db-drop.sh create mode 100755 tutorials/Oracle-to-Debezium-olr/2.db-start.sh create mode 100755 tutorials/Oracle-to-Debezium-olr/3.kafka-start.sh create mode 100755 tutorials/Oracle-to-Debezium-olr/4.olr-start.sh create mode 100755 tutorials/Oracle-to-Debezium-olr/5.debezium-start.sh create mode 100755 tutorials/Oracle-to-Debezium-olr/6.test.sh create mode 100755 tutorials/Oracle-to-Debezium-olr/7.debezium-drop.sh create mode 100755 tutorials/Oracle-to-Debezium-olr/8.olr-drop.sh create mode 100755 tutorials/Oracle-to-Debezium-olr/9.kafka-drop.sh create mode 100644 tutorials/Oracle-to-Debezium-olr/README.md create mode 100755 tutorials/Oracle-to-Debezium-olr/cfg.sh create mode 100644 tutorials/Oracle-to-Debezium-olr/compose.yaml create mode 100644 tutorials/Oracle-to-Debezium-olr/scripts/OpenLogReplicator.json create mode 100644 tutorials/Oracle-to-Debezium-olr/setup/config.sql create mode 100644 tutorials/Oracle-to-Debezium-olr/sql/drop-usrdbz_olr.sql create mode 100644 tutorials/Oracle-to-Debezium-olr/sql/drop-usrolr.sql create mode 100644 tutorials/Oracle-to-Debezium-olr/sql/schema-usrdbz_olr.sql create mode 100644 tutorials/Oracle-to-Debezium-olr/sql/schema-usrolr.sql create mode 100644 tutorials/Oracle-to-Debezium-olr/sql/schema-usrtbl.sql create mode 100644 tutorials/Oracle-to-Debezium-olr/sql/test.sql diff --git a/README.md b/README.md index d9eafc2..3e70b9c 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,9 @@ The following custom [Docker images](images/README.md) are required to run tutor - `oracle/database:21.3.0-xe` - Oracle Database 21.3 XE - `bersler/openlogreplicator:tutorial` - OpenLogReplicator +And containers for Debezium/Kafka ecosystem: +- `quay.io/debezium/kafka` - Kafka +- `quay.io/debezium/connect` - Debezium connector Before running the tutorials, create the necessary Docker images. Not all images are required for all tutorials. @@ -43,15 +46,21 @@ Refer to the particular tutorial for the list of required images. ## Tutorials -### 1. [Oracle-to-file](tutorials/Oracle-to-file/README.md) +### 1. [Oracle-to-Debezium-logminer](tutorials/Oracle-to-Debezium-logminer/README.md) + +Goal: demonstrate capturing transactional changes from a database and writing them to a Kafka topic, covering setup, a test change that produces the output, and cleanup. +Uses Debezium and LogMiner as the capture method. +OpenLogReplicator is not used. + +### 2. [Oracle-to-file](tutorials/Oracle-to-file/README.md) Goal: demonstrate capturing transactional changes from a database and writing them to a file, covering setup, a test change that produces the output, and cleanup. -### 2. [Oracle-to-file-offline](tutorials/Oracle-to-file-offline/README.md) +### 3. [Oracle-to-file-offline](tutorials/Oracle-to-file-offline/README.md) Goal: demonstrate capturing transactional changes from a database and writing them to a file without a direct database connection, covering setup, a test change that produces the output, and cleanup. -### 3. [Oracle-to-Kafka](tutorials/Oracle-to-Kafka/README.md) +### 4. [Oracle-to-Kafka](tutorials/Oracle-to-Kafka/README.md) Goal: demonstrate capturing transactional changes from a database and writing them to a Kafka topic, covering setup, a test change that produces the output, and cleanup. diff --git a/tutorials/Oracle-to-Debezium-both/1.check.sh b/tutorials/Oracle-to-Debezium-both/1.check.sh new file mode 100755 index 0000000..93e8fd1 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/1.check.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "1. checking docker images" +docker_check "${DB_IMAGE}" +docker_check "${OLR_IMAGE}" +finish diff --git a/tutorials/Oracle-to-Debezium-both/10.db-drop.sh b/tutorials/Oracle-to-Debezium-both/10.db-drop.sh new file mode 100755 index 0000000..1c83cd1 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/10.db-drop.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "10. dropping database container" +docker_rm ${DB_CONTAINER} +docker_down +file_rm fra oradata oracle_libs sql/schema-usrtbl.out "${INSTANT_CLIENT_DIR}" +finish diff --git a/tutorials/Oracle-to-Debezium-both/2.db-start.sh b/tutorials/Oracle-to-Debezium-both/2.db-start.sh new file mode 100755 index 0000000..dceb923 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/2.db-start.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "2. creating and starting database container" +db_files "oracle_libs" +docker_up +db_wait "${DB_CONTAINER}" +db_sql "${DB_CONTAINER}" /opt/sql/schema-usrtbl.sql /opt/sql/schema-usrtbl.out +db_client +finish diff --git a/tutorials/Oracle-to-Debezium-both/3.kafka-start.sh b/tutorials/Oracle-to-Debezium-both/3.kafka-start.sh new file mode 100755 index 0000000..a333e78 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/3.kafka-start.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "3. creating and starting Kafka container" +kafka_files +docker_up_wait --profile kafka +kafka_create_topic_compact "${KAFKA_CONTAINER}" "${KAFKA_BROKER}" "${KAFKA_TOPIC_CONFIGS}" +kafka_create_topic_compact "${KAFKA_CONTAINER}" "${KAFKA_BROKER}" "${KAFKA_TOPIC_OFFSETS}" +kafka_create_topic_compact "${KAFKA_CONTAINER}" "${KAFKA_BROKER}" "${KAFKA_TOPIC_STATUSSES}" +kafka_list_topics "${KAFKA_CONTAINER}" "${KAFKA_BROKER}" +finish diff --git a/tutorials/Oracle-to-Debezium-both/4.olr-start.sh b/tutorials/Oracle-to-Debezium-both/4.olr-start.sh new file mode 100755 index 0000000..2182f2e --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/4.olr-start.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "4. creating and starting OpenLogReplicator container" +olr_files +db_sql "${DB_CONTAINER}" /opt/sql/schema-usrolr.sql /opt/sql/schema-usrolr.out +docker_up --profile openlogreplicator --profile kafka +# olr_wait_for_start "${OLR_CONTAINER}" "ready" +sleep 10 +finish diff --git a/tutorials/Oracle-to-Debezium-both/5.debezium-start.sh b/tutorials/Oracle-to-Debezium-both/5.debezium-start.sh new file mode 100755 index 0000000..8cec7cf --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/5.debezium-start.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "5. creating and starting Debezium container" +db_sql "${DB_CONTAINER}" /opt/sql/schema-usrdbz_logminer.sql /opt/sql/schema-usrdbz_logminer.out +db_sql "${DB_CONTAINER}" /opt/sql/schema-usrdbz_olr.sql /opt/sql/schema-usrdbz_olr.out +sudo chown 1001:1001 ~/debezium/debezium-connector-oracle-3.4.0.Final.jar +docker_up_wait --profile debezium --profile kafka +debezium_olr "${DEBEZIUM_CONTAINER}" "${KAFKA_BROKER}" +debezium_wait_for_connect "${DEBEZIUM_CONTAINER}" "oracle-olr-connector" +debezium_logminer "${DEBEZIUM_CONTAINER}" "${KAFKA_BROKER}" +debezium_wait_for_connect "${DEBEZIUM_CONTAINER}" "oracle-logminer-connector" +finish diff --git a/tutorials/Oracle-to-Debezium-both/6.test.sh b/tutorials/Oracle-to-Debezium-both/6.test.sh new file mode 100755 index 0000000..013618e --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/6.test.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "6. running test" +db_sql "${DB_CONTAINER}" /opt/sql/test.sql /opt/sql/test.out +kafka_wait_for_messages "${KAFKA_CONTAINER}" "${KAFKA_BROKER}" "${KAFKA_TOPIC_OLR}" "COUNT" 5 +kafka_wait_for_messages "${KAFKA_CONTAINER}" "${KAFKA_BROKER}" "${KAFKA_TOPIC_LOGMINER}" "COUNT" 5 +finish diff --git a/tutorials/Oracle-to-Debezium-both/7.debezium-drop.sh b/tutorials/Oracle-to-Debezium-both/7.debezium-drop.sh new file mode 100755 index 0000000..21de3e1 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/7.debezium-drop.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "7. dropping Debezium container" +docker_rm "${DEBEZIUM_CONTAINER}" +db_sql "${DB_CONTAINER}" /opt/sql/drop-usrdbz_olr.sql /opt/sql/drop-usrdbz_olr.out +db_sql "${DB_CONTAINER}" /opt/sql/drop-usrdbz_logminer.sql /opt/sql/drop-usrdbz_logminer.out +file_rm sql/schema-usrdbz_olr.out sql/drop-usrdbz_olr.out +file_rm sql/schema-usrdbz_logminer.out sql/drop-usrdbz_logminer.out +finish diff --git a/tutorials/Oracle-to-Debezium-both/8.olr-drop.sh b/tutorials/Oracle-to-Debezium-both/8.olr-drop.sh new file mode 100755 index 0000000..80a3bc1 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/8.olr-drop.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "8. dropping OpenLogReplicator container" +docker_rm "${OLR_CONTAINER}" +db_sql "${DB_CONTAINER}" /opt/sql/drop-usrolr.sql /opt/sql/drop-usrolr.out +file_rm sql/drop-usrolr.out sql/schema-usrolr.out sql/test.out checkpoint log +finish diff --git a/tutorials/Oracle-to-Debezium-both/9.kafka-drop.sh b/tutorials/Oracle-to-Debezium-both/9.kafka-drop.sh new file mode 100755 index 0000000..efd5903 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/9.kafka-drop.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "9. dropping Kafka container" +docker_rm "${KAFKA_CONTAINER}" +file_rm kafka +finish diff --git a/tutorials/Oracle-to-Debezium-both/README.md b/tutorials/Oracle-to-Debezium-both/README.md new file mode 100644 index 0000000..b65db31 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/README.md @@ -0,0 +1,116 @@ +# Replicating Oracle transactions to Kafka with Debezium + +Goal: reference replication with Debezium by capturing transactional changes from a database and delivering them to a Kafka topic, covering environment setup, enabling replication, executing a test change that produces the output, and cleanup. +Uses OpenLogReplicator as the capture method. + +## Create Docker images + +The following images are required for this tutorial: +- `oracle/database:21.3.0-xe` - Oracle Database 21.3 XE +- `quay.io/debezium/kafka` - Kafka +- `quay.io/debezium/connect` - Debezium connector + +Refer to [Docker images](../../images/README.md) for image creation scripts. + +## Container names + +The default container names are: +- `ORA1` - Oracle Database 21.3 XE +- `KAFKA1` - Kafka broker +- `DEBEZIUM1` - Debezium connector + +For ease of use, a Zookeeper container is not created separately, since a newer Kafka version is used which relies on KRaft. +Container names are defined in the `cfg.sh` script. + +## Folders used by this tutorial + +All files for the database devices and redo log files are placed outside the containers. +The folders are created in steps 2-3-4 and cleaned up in steps 6-7-8. +The folders created by `*start.sh` scripts are removed by `*drop.sh` scripts. + +## Run test + +### 1. Check if images are correct. + +Run: + +---- + ./1.check.sh +---- + +### 2. Create the database and run all schema scripts. + +This step represents the state that is usually present in the enterprise. +The database is created, and some applications are installed. + +Run: + +---- + ./2.db-start.sh +---- + +### 3. Create Kafka container and topic. + +During this step, the Kafka container is created and started. +Also, the topic for OpenLogReplicator is created. + +Run: + +---- + ./3.kafka-start.sh +---- + +### 4. Create and start Debezium using OpenLogReplicator. + +This step represents the state when replication is added to the database. + +Run: + +---- + ./4.debezium-start.sh +---- + +After this step, the container with Debezium using OpenLogReplicator should be up and running. +Transactions are being read from the database and sent to Kafka. +Replication is active. + +### 5. Run SQL and check results + +This step represents a business event when the database is changed and changes are captured by OpenLogReplicator and published by Debezium connector. + +Run: + +---- + ./5.test.sh +---- + +The expected result is that the Kafka topic contains messages with the JSON representation of the transaction executed by the `sql/test.sql` script. + +### 6. Drop Debezium + +All tests are finished. +Replication can be removed. + +Run: + +---- + ./6.debezium-drop.sh +---- + +### 7. Drop Kafka + +Run: + +---- + ./7.kafka-drop.sh +---- + +After this step, you may go back to step 3 and repeat the process. + +### 8. Drop the database + +Run: + +---- + ./8.db-drop.sh +---- diff --git a/tutorials/Oracle-to-Debezium-both/cfg.sh b/tutorials/Oracle-to-Debezium-both/cfg.sh new file mode 100755 index 0000000..343d8b2 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/cfg.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . + +export DB_IMAGE=${DB_IMAGE:=oracle/database:21.3.0-xe} +export DB_CONTAINER=${DB_CONTAINER:=ORA1} +export KAFKA_VERSION=3.4.0.Final +export KAFKA_IMAGE=${KAFKA_IMAGE:=quay.io/debezium/kafka:${KAFKA_VERSION}} +export KAFKA_CLUSTER=${KAFKA_CLUSTER:=KAFKA1} +export KAFKA_CONTAINER=${KAFKA_CONTAINER:=KAFKA1} +export KAFKA_BROKER=${KAFKA_BROKER:=kafka:9092} +export KAFKA_TOPIC_LOGMINER=${KAFKA_TOPIC_LOGMINER:=oracle_logminer.USRTBL.ADAM1} +export KAFKA_TOPIC_CONFIGS=${KAFKA_TOPIC_CONFIGS:=topic_configs} +export KAFKA_TOPIC_OFFSETS=${KAFKA_TOPIC_OFFSETS:=topic_offsets} +export KAFKA_TOPIC_STATUSSES=${KAFKA_TOPIC_STATUSSES:=topic_statuses} +export KAFKA_CONTAINER=${KAFKA_CONTAINER:=KAFKA1} +export KAFKA_BROKER=${KAFKA_BROKER:=kafka:9092} +export KAFKA_TOPIC_OLR=${KAFKA_TOPIC_OLR:=oracle_olr.USRTBL.ADAM1} +export OLR_IMAGE=${OLR_IMAGE:=bersler/openlogreplicator:tutorial} +export OLR_CONTAINER=${OLR_CONTAINER:=OLR1} +export OLR_CUSTOM=${OLR_CUSTOM:=} +export DEBEZIUM_VERSION=3.4.0.Final +export DEBEZIUM_IMAGE=${DEBEZIUM_IMAGE:=quay.io/debezium/connect:${DEBEZIUM_VERSION}} +export DEBEZIUM_CONTAINER=${DEBEZIUM_CONTAINER:=DEBEZIUM1} +export DEBEZIUM_CONTAINER=${DEBEZIUM_CONTAINER:=DEBEZIUM1} +export JDBC_FILE=ojdbc8.jar +export JDBC_URL=https://download.oracle.com/otn-pub/otn_software/jdbc/2326/ojdbc8.jar +export INSTANT_CLIENT_FILE=instantclient-basic-linux.x64-23.26.0.0.0.zip +export INSTANT_CLIENT_URL=https://download.oracle.com/otn_software/linux/instantclient/2326000/instantclient-basic-linux.x64-23.26.0.0.0.zip +export INSTANT_CLIENT_DIR=instantclient_23_26 +export DUMP_LOGS=${DUMP_LOGS:=0} diff --git a/tutorials/Oracle-to-Debezium-both/compose.yaml b/tutorials/Oracle-to-Debezium-both/compose.yaml new file mode 100644 index 0000000..231537b --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/compose.yaml @@ -0,0 +1,120 @@ +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . + +services: + oracle: + image: "${DB_IMAGE}" + container_name: "${DB_CONTAINER}" + volumes: + - ./fra:/opt/oracle/fra + - ./oradata:/opt/oracle/oradata + - ./sql:/opt/sql + - ./setup:/opt/oracle/scripts/setup + networks: + - isolated_network + environment: + - ORACLE_CHARACTERSET=AL32UTF8 + build: + context: . + shm_size: '2gb' + shm_size: '2gb' + + kafka: + image: "${KAFKA_IMAGE}" + container_name: "${KAFKA_CONTAINER}" + profiles: ["kafka"] + volumes: + - ./kafka:/var/lib/kafka/data + networks: + - isolated_network + environment: + # KRaft parameters + - CLUSTER_ID=${KAFKA_CLUSTER} + - KAFKA_NODE_ID=1 + - KAFKA_PROCESS_ROLES=broker,controller + - KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS=kafka:9093 + # Listeners + - KAFKA_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093 + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092 + - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT + # Performance & configuration + - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1 + - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1 + healthcheck: + test: ["CMD", "/kafka/bin/kafka-topics.sh", "--bootstrap-server", "${KAFKA_BROKER}", "--list"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 60s + + openlogreplicator: + image: "${OLR_IMAGE}" + container_name: "${OLR_CONTAINER}" + profiles: ["openlogreplicator"] + depends_on: + - oracle + - kafka + volumes: + - ./bin:/opt/bin + - ./checkpoint:/opt/OpenLogReplicator/checkpoint + - ./dump:/opt/dump + - ./fra:/opt/fra + - ./log:/opt/OpenLogReplicator/log + - ./oradata:/opt/oradata + - ./scripts:/opt/OpenLogReplicator/scripts + restart: "no" + networks: + - isolated_network + + connect: + image: "${DEBEZIUM_IMAGE}" + container_name: "${DEBEZIUM_CONTAINER}" + profiles: ["debezium"] + depends_on: + - kafka + environment: + - BOOTSTRAP_SERVERS=kafka:9092 + - GROUP_ID=1 + - CONFIG_STORAGE_TOPIC=${KAFKA_TOPIC_CONFIGS} + - OFFSET_STORAGE_TOPIC=${KAFKA_TOPIC_OFFSETS} + - STATUS_STORAGE_TOPIC=${KAFKA_TOPIC_STATUSSES} + # Debezium specific settings + - CONFIG_STORAGE_REPLICATION_FACTOR=1 + - OFFSET_STORAGE_REPLICATION_FACTOR=1 + - STATUS_STORAGE_REPLICATION_FACTOR=1 + # Oracle connector specific settings + - LD_LIBRARY_PATH=/instant_client + volumes: + - ./oracle_libs/${JDBC_FILE}:/kafka/connect/debezium-connector-oracle/${JDBC_FILE} + - ./${INSTANT_CLIENT_DIR}:/instant_client + - ~/debezium/debezium-connector-oracle-3.4.0.Final.jar:/kafka/connect/debezium-connector-oracle/debezium-connector-oracle-3.4.0.Final.jar + networks: + - isolated_network + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:8083/" ] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + +networks: + isolated_network: + internal: true diff --git a/tutorials/Oracle-to-Debezium-both/scripts/OpenLogReplicator.json b/tutorials/Oracle-to-Debezium-both/scripts/OpenLogReplicator.json new file mode 100644 index 0000000..d07a696 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/scripts/OpenLogReplicator.json @@ -0,0 +1,43 @@ +{ + "version": "2.0.0", + "log-level": 3, + "trace": 524288, + "dump-redo-log": 2, + "dump-raw-data": 1, + "dump-path": "/opt/dump", + "metrics": { + "type": "prometheus", + "bind": "0.0.0.0:8080" + }, + "source": [ + { + "alias": "S1", + "name": "ORA1", + "reader": { + "type": "online", + "path-mapping": ["/opt/oracle/oradata", "/opt/oradata", "/opt/oracle/fra", "/opt/fra"], + "user": "USROLR", + "password": "USROLRPWD", + "server": "//oracle:1521/XEPDB1" + }, + "format": { + "type": "debezium" + }, + "filter": { + "table": [ + {"owner": "USRTBL", "table": "ADAM.*"} + ] + } + } + ], + "target": [ + { + "alias": "K1", + "source": "S1", + "writer": { + "type": "network", + "uri": "0.0.0.0:5000" + } + } + ] +} diff --git a/tutorials/Oracle-to-Debezium-both/setup/config.sql b/tutorials/Oracle-to-Debezium-both/setup/config.sql new file mode 100644 index 0000000..780a38d --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/setup/config.sql @@ -0,0 +1,10 @@ +SHUTDOWN IMMEDIATE; +STARTUP MOUNT; +ALTER DATABASE ARCHIVELOG; +ALTER DATABASE OPEN; +ALTER DATABASE ADD SUPPLEMENTAL LOG DATA; +ALTER SYSTEM ARCHIVE LOG CURRENT; + +ALTER SYSTEM SET DB_CREATE_FILE_DEST = '/opt/oracle/oradata' SCOPE = BOTH; +ALTER SYSTEM SET DB_RECOVERY_FILE_DEST_SIZE = '250G' SCOPE = BOTH; +ALTER SYSTEM SET DB_RECOVERY_FILE_DEST = '/opt/oracle/fra' SCOPE = BOTH; diff --git a/tutorials/Oracle-to-Debezium-both/sql/drop-usrdbz_logminer.sql b/tutorials/Oracle-to-Debezium-both/sql/drop-usrdbz_logminer.sql new file mode 100644 index 0000000..b260dff --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/sql/drop-usrdbz_logminer.sql @@ -0,0 +1,7 @@ +ALTER SESSION SET CONTAINER = CDB$ROOT; +DROP USER C##USRDBZ_LOGMINER CASCADE; + +ALTER SESSION SET CONTAINER = XEPDB1; +DROP TABLESPACE DBZTBLSPCLOGMINER; +ALTER SESSION SET CONTAINER = CDB$ROOT; +DROP TABLESPACE DBZTBLSPCLOGMINER; diff --git a/tutorials/Oracle-to-Debezium-both/sql/drop-usrdbz_olr.sql b/tutorials/Oracle-to-Debezium-both/sql/drop-usrdbz_olr.sql new file mode 100644 index 0000000..3d93c70 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/sql/drop-usrdbz_olr.sql @@ -0,0 +1,7 @@ +ALTER SESSION SET CONTAINER = CDB$ROOT; +DROP USER C##USRDBZ_OLR CASCADE; + +ALTER SESSION SET CONTAINER = XEPDB1; +DROP TABLESPACE DBZTBLSPCOLR; +ALTER SESSION SET CONTAINER = CDB$ROOT; +DROP TABLESPACE DBZTBLSPCOLR; diff --git a/tutorials/Oracle-to-Debezium-both/sql/drop-usrolr.sql b/tutorials/Oracle-to-Debezium-both/sql/drop-usrolr.sql new file mode 100644 index 0000000..5c61315 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/sql/drop-usrolr.sql @@ -0,0 +1 @@ +DROP USER USROLR CASCADE; diff --git a/tutorials/Oracle-to-Debezium-both/sql/schema-usrdbz_logminer.sql b/tutorials/Oracle-to-Debezium-both/sql/schema-usrdbz_logminer.sql new file mode 100644 index 0000000..a08f816 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/sql/schema-usrdbz_logminer.sql @@ -0,0 +1,34 @@ +CREATE TABLESPACE DBZTBLSPCLOGMINER DATAFILE SIZE 25M AUTOEXTEND ON MAXSIZE UNLIMITED; + +ALTER SESSION SET CONTAINER = CDB$ROOT; +CREATE TABLESPACE DBZTBLSPCLOGMINER DATAFILE SIZE 25M AUTOEXTEND ON MAXSIZE UNLIMITED; + +ALTER SESSION SET CONTAINER = CDB$ROOT; +CREATE USER C##USRDBZ_LOGMINER IDENTIFIED BY dbzpwd + DEFAULT TABLESPACE DBZTBLSPCLOGMINER + QUOTA UNLIMITED ON DBZTBLSPCLOGMINER + CONTAINER = ALL; + +ALTER SESSION SET CONTAINER = CDB$ROOT; +GRANT CREATE SESSION TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT CREATE TABLE TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT CREATE SEQUENCE TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT SET CONTAINER TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT SELECT_CATALOG_ROLE TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT EXECUTE_CATALOG_ROLE TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT SELECT ANY TRANSACTION TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT SELECT ANY DICTIONARY TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT SELECT ANY TABLE TO C##USRDBZ_LOGMINER CONTAINER=ALL; + +GRANT LOGMINING TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT EXECUTE ON DBMS_LOGMNR TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT EXECUTE ON DBMS_LOGMNR_D TO C##USRDBZ_LOGMINER CONTAINER=ALL; + +GRANT SELECT ON V_$LOG TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT SELECT ON V_$LOGFILE TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT SELECT ON V_$LOGMNR_CONTENTS TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT SELECT ON V_$ARCHIVED_LOG TO C##USRDBZ_LOGMINER CONTAINER=ALL; + +ALTER SESSION SET CONTAINER = XEPDB1; +GRANT FLASHBACK ON USRTBL.ADAM1 TO C##USRDBZ_LOGMINER; +GRANT SELECT ON USRTBL.ADAM1 TO C##USRDBZ_LOGMINER; diff --git a/tutorials/Oracle-to-Debezium-both/sql/schema-usrdbz_olr.sql b/tutorials/Oracle-to-Debezium-both/sql/schema-usrdbz_olr.sql new file mode 100644 index 0000000..98ba3ee --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/sql/schema-usrdbz_olr.sql @@ -0,0 +1,30 @@ +CREATE TABLESPACE DBZTBLSPCOLR DATAFILE SIZE 25M AUTOEXTEND ON MAXSIZE UNLIMITED; + +ALTER SESSION SET CONTAINER = CDB$ROOT; +CREATE TABLESPACE DBZTBLSPCOLR DATAFILE SIZE 25M AUTOEXTEND ON MAXSIZE UNLIMITED; + +ALTER SESSION SET CONTAINER = CDB$ROOT; +CREATE USER C##USRDBZ_OLR IDENTIFIED BY dbzpwd + DEFAULT TABLESPACE DBZTBLSPCOLR + QUOTA UNLIMITED ON DBZTBLSPCOLR + CONTAINER = ALL; + +ALTER SESSION SET CONTAINER = CDB$ROOT; +GRANT CREATE SESSION TO C##USRDBZ_OLR CONTAINER=ALL; +GRANT CREATE TABLE TO C##USRDBZ_OLR CONTAINER=ALL; +GRANT CREATE SEQUENCE TO C##USRDBZ_OLR CONTAINER=ALL; +GRANT SET CONTAINER TO C##USRDBZ_OLR CONTAINER=ALL; +GRANT SELECT_CATALOG_ROLE TO C##USRDBZ_OLR CONTAINER=ALL; +GRANT EXECUTE_CATALOG_ROLE TO C##USRDBZ_OLR CONTAINER=ALL; +GRANT SELECT ANY TRANSACTION TO C##USRDBZ_OLR CONTAINER=ALL; +GRANT SELECT ANY DICTIONARY TO C##USRDBZ_OLR CONTAINER=ALL; +GRANT SELECT ANY TABLE TO C##USRDBZ_OLR CONTAINER=ALL; + +GRANT SELECT ON V_$LOG TO C##USRDBZ_OLR CONTAINER=ALL; +GRANT SELECT ON V_$LOGFILE TO C##USRDBZ_OLR CONTAINER=ALL; +GRANT SELECT ON V_$LOGMNR_CONTENTS TO C##USRDBZ_OLR CONTAINER=ALL; +GRANT SELECT ON V_$ARCHIVED_LOG TO C##USRDBZ_OLR CONTAINER=ALL; + +ALTER SESSION SET CONTAINER = XEPDB1; +GRANT FLASHBACK ON USRTBL.ADAM1 TO C##USRDBZ_OLR; +GRANT SELECT ON USRTBL.ADAM1 TO C##USRDBZ_OLR; diff --git a/tutorials/Oracle-to-Debezium-both/sql/schema-usrolr.sql b/tutorials/Oracle-to-Debezium-both/sql/schema-usrolr.sql new file mode 100644 index 0000000..c1bc4d9 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/sql/schema-usrolr.sql @@ -0,0 +1,46 @@ +CREATE USER USROLR IDENTIFIED BY USROLRPWD TEMPORARY TABLESPACE TEMP; + +GRANT CONNECT TO USROLR; +GRANT RESOURCE TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.CCOL$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.CDEF$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.COL$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.DEFERRED_STG$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.ECOL$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.LOB$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.LOBCOMPPART$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.LOBFRAG$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.OBJ$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.TAB$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.TABCOMPART$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.TABPART$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.TABSUBPART$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.TS$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.USER$ TO USROLR; +GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO USROLR; +GRANT SELECT ON SYS.V_$DATABASE TO USROLR; +GRANT SELECT ON SYS.V_$DATABASE_INCARNATION TO USROLR; +GRANT SELECT ON SYS.V_$LOG TO USROLR; +GRANT SELECT ON SYS.V_$LOGFILE TO USROLR; +GRANT SELECT ON SYS.V_$PARAMETER TO USROLR; +GRANT SELECT ON SYS.V_$PDBS TO USROLR; +GRANT SELECT ON SYS.V_$STANDBY_LOG TO USROLR; +GRANT SELECT ON SYS.V_$TRANSPORTABLE_PLATFORM TO USROLR; +GRANT SELECT, FLASHBACK ON XDB.XDB$TTSET TO USROLR; +DECLARE +CURSOR C1 IS SELECT TOKSUF FROM XDB.XDB$TTSET; +CMD VARCHAR2(2000); +BEGIN +FOR C IN C1 LOOP + CMD := 'GRANT SELECT, FLASHBACK ON XDB.X$NM' || C.TOKSUF || ' TO USROLR'; +EXECUTE IMMEDIATE CMD; +CMD := 'GRANT SELECT, FLASHBACK ON XDB.X$QN' || C.TOKSUF || ' TO USROLR'; +EXECUTE IMMEDIATE CMD; +CMD := 'GRANT SELECT, FLASHBACK ON XDB.X$PT' || C.TOKSUF || ' TO USROLR'; +EXECUTE IMMEDIATE CMD; +END LOOP; +END; +/ + +ALTER SESSION SET CONTAINER = CDB$ROOT; +ALTER SYSTEM ARCHIVE LOG CURRENT; diff --git a/tutorials/Oracle-to-Debezium-both/sql/schema-usrtbl.sql b/tutorials/Oracle-to-Debezium-both/sql/schema-usrtbl.sql new file mode 100644 index 0000000..651fda7 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/sql/schema-usrtbl.sql @@ -0,0 +1,27 @@ +ALTER SESSION SET CONTAINER = CDB$ROOT; +ALTER SYSTEM ARCHIVE LOG CURRENT; + +ALTER SESSION SET CONTAINER = XEPDB1; +CREATE TABLESPACE TBLS1 DATAFILE SIZE 100M AUTOEXTEND ON NEXT 100M; +ALTER TABLESPACE TBLS1 FORCE LOGGING; + +CREATE USER USRTBL IDENTIFIED BY USRTBLPWD DEFAULT TABLESPACE TBLS1 TEMPORARY TABLESPACE TEMP; +ALTER USER USRTBL QUOTA UNLIMITED ON TBLS1; + +GRANT CONNECT TO USRTBL; +GRANT RESOURCE TO USRTBL; + +CREATE TABLE USRTBL.ADAM1( + ID NUMBER NOT NULL, + NAME VARCHAR2(30), + COUNT NUMBER, + START_TIME0 TIMESTAMP(0), + START_TIME3 TIMESTAMP(3), + START_TIME6 TIMESTAMP(6), + START_TIME9 TIMESTAMP(9) +); +ALTER TABLE USRTBL.ADAM1 ADD CONSTRAINT ADAM1PK PRIMARY KEY(ID); +ALTER TABLE USRTBL.ADAM1 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER SESSION SET CONTAINER = CDB$ROOT; +ALTER SYSTEM ARCHIVE LOG CURRENT; diff --git a/tutorials/Oracle-to-Debezium-both/sql/test.sql b/tutorials/Oracle-to-Debezium-both/sql/test.sql new file mode 100644 index 0000000..37675c4 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-both/sql/test.sql @@ -0,0 +1,12 @@ +INSERT INTO USRTBL.ADAM1 VALUES (1, 'Adam Leszczyński', 10, + TO_TIMESTAMP('2019-08-01 12:34:56', 'YYYY-MM-DD HH24:MI:SS'), + TO_TIMESTAMP('2019-08-01 12:34:56.123', 'YYYY-MM-DD HH24:MI:SS.FF'), + TO_TIMESTAMP('2019-08-01 12:34:56.123456', 'YYYY-MM-DD HH24:MI:SS.FF'), + TO_TIMESTAMP('2019-08-01 12:34:56.123456789', 'YYYY-MM-DD HH24:MI:SS.FF')); +COMMIT; +UPDATE USRTBL.ADAM1 SET COUNT = COUNT + 1; +UPDATE USRTBL.ADAM1 SET COUNT = COUNT + 1; +UPDATE USRTBL.ADAM1 SET COUNT = COUNT + 1; +COMMIT; +DELETE FROM USRTBL.ADAM1; +COMMIT; diff --git a/tutorials/Oracle-to-Debezium-logminer/1.check.sh b/tutorials/Oracle-to-Debezium-logminer/1.check.sh new file mode 100755 index 0000000..4ffc898 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-logminer/1.check.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "1. checking docker images" +docker_check "${DB_IMAGE}" +finish diff --git a/tutorials/Oracle-to-Debezium-logminer/2.db-start.sh b/tutorials/Oracle-to-Debezium-logminer/2.db-start.sh new file mode 100755 index 0000000..dceb923 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-logminer/2.db-start.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "2. creating and starting database container" +db_files "oracle_libs" +docker_up +db_wait "${DB_CONTAINER}" +db_sql "${DB_CONTAINER}" /opt/sql/schema-usrtbl.sql /opt/sql/schema-usrtbl.out +db_client +finish diff --git a/tutorials/Oracle-to-Debezium-logminer/3.kafka-start.sh b/tutorials/Oracle-to-Debezium-logminer/3.kafka-start.sh new file mode 100755 index 0000000..a333e78 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-logminer/3.kafka-start.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "3. creating and starting Kafka container" +kafka_files +docker_up_wait --profile kafka +kafka_create_topic_compact "${KAFKA_CONTAINER}" "${KAFKA_BROKER}" "${KAFKA_TOPIC_CONFIGS}" +kafka_create_topic_compact "${KAFKA_CONTAINER}" "${KAFKA_BROKER}" "${KAFKA_TOPIC_OFFSETS}" +kafka_create_topic_compact "${KAFKA_CONTAINER}" "${KAFKA_BROKER}" "${KAFKA_TOPIC_STATUSSES}" +kafka_list_topics "${KAFKA_CONTAINER}" "${KAFKA_BROKER}" +finish diff --git a/tutorials/Oracle-to-Debezium-logminer/4.debezium-start.sh b/tutorials/Oracle-to-Debezium-logminer/4.debezium-start.sh new file mode 100755 index 0000000..26c604d --- /dev/null +++ b/tutorials/Oracle-to-Debezium-logminer/4.debezium-start.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "4. creating and starting Debezium container" +db_sql "${DB_CONTAINER}" /opt/sql/schema-usrdbz_logminer.sql /opt/sql/schema-usrdbz_logminer.out +docker_up_wait --profile debezium --profile kafka +debezium_logminer "${DEBEZIUM_CONTAINER}" "${KAFKA_BROKER}" +debezium_wait_for_connect "${DEBEZIUM_CONTAINER}" "oracle-logminer-connector" +finish diff --git a/tutorials/Oracle-to-Debezium-logminer/5.test.sh b/tutorials/Oracle-to-Debezium-logminer/5.test.sh new file mode 100755 index 0000000..3ebe9df --- /dev/null +++ b/tutorials/Oracle-to-Debezium-logminer/5.test.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "5. running test - verify data in Kafka topic" +db_sql "${DB_CONTAINER}" /opt/sql/test.sql /opt/sql/test.out +kafka_wait_for_messages "${KAFKA_CONTAINER}" "${KAFKA_BROKER}" "${KAFKA_TOPIC_LOGMINER}" "COUNT" 3 + +echo "5. running test - restart database" +docker stop ${DB_CONTAINER} +docker start ${DB_CONTAINER} +db_wait "${DB_CONTAINER}" +sleep 30 +db_sql "${DB_CONTAINER}" /opt/sql/test.sql /opt/sql/test.out +kafka_wait_for_messages "${KAFKA_CONTAINER}" "${KAFKA_BROKER}" "${KAFKA_TOPIC_LOGMINER}" "COUNT" 6 + +echo "5. running test - restart Debezium" +docker stop ${DEBEZIUM_CONTAINER} +docker start ${DEBEZIUM_CONTAINER} +debezium_wait_for_connect "${DEBEZIUM_CONTAINER}" "oracle-logminer-connector" +db_sql "${DB_CONTAINER}" /opt/sql/test.sql /opt/sql/test.out +kafka_wait_for_messages "${KAFKA_CONTAINER}" "${KAFKA_BROKER}" "${KAFKA_TOPIC_LOGMINER}" "COUNT" 9 + + +finish diff --git a/tutorials/Oracle-to-Debezium-logminer/6.debezium-drop.sh b/tutorials/Oracle-to-Debezium-logminer/6.debezium-drop.sh new file mode 100755 index 0000000..37fabab --- /dev/null +++ b/tutorials/Oracle-to-Debezium-logminer/6.debezium-drop.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "6. dropping Debezium container" +docker_rm "${DEBEZIUM_CONTAINER}" +db_sql "${DB_CONTAINER}" /opt/sql/drop-usrdbz_logminer.sql /opt/sql/drop-usrdbz_logminer.out +file_rm sql/schema-usrdbz_logminer.out sql/drop-usrdbz_logminer.out +finish diff --git a/tutorials/Oracle-to-Debezium-logminer/7.kafka-drop.sh b/tutorials/Oracle-to-Debezium-logminer/7.kafka-drop.sh new file mode 100755 index 0000000..1d9c9e1 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-logminer/7.kafka-drop.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "7. dropping Kafka container" +docker_rm "${KAFKA_CONTAINER}" +file_rm kafka +finish diff --git a/tutorials/Oracle-to-Debezium-logminer/8.db-drop.sh b/tutorials/Oracle-to-Debezium-logminer/8.db-drop.sh new file mode 100755 index 0000000..c1638d1 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-logminer/8.db-drop.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "8. dropping database container" +docker_rm ${DB_CONTAINER} +docker_down +file_rm fra oradata oracle_libs sql/schema-usrtbl.out "${INSTANT_CLIENT_DIR}" +finish diff --git a/tutorials/Oracle-to-Debezium-logminer/README.md b/tutorials/Oracle-to-Debezium-logminer/README.md new file mode 100644 index 0000000..aa4277c --- /dev/null +++ b/tutorials/Oracle-to-Debezium-logminer/README.md @@ -0,0 +1,117 @@ +# Replicating Oracle transactions to Kafka with Debezium + +Goal: reference replication with Debezium by capturing transactional changes from a database and delivering them to a Kafka topic, covering environment setup, enabling replication, executing a test change that produces the output, and cleanup. +Uses LogMiner as the capture method. +OpenLogReplicator is not used. + +## Create Docker images + +The following images are required for this tutorial: +- `oracle/database:21.3.0-xe` - Oracle Database 21.3 XE +- `quay.io/debezium/kafka` - Kafka +- `quay.io/debezium/connect` - Debezium connector + +Refer to [Docker images](../../images/README.md) for image creation scripts. + +## Container names + +The default container names are: +- `ORA1` - Oracle Database 21.3 XE +- `KAFKA1` - Kafka broker +- `DEBEZIUM1` - Debezium connector + +For ease of use, a Zookeeper container is not created separately, since a newer Kafka version is used which relies on KRaft. +Container names are defined in the `cfg.sh` script. + +## Folders used by this tutorial + +All files for the database devices and redo log files are placed outside the containers. +The folders are created in steps 2-3-4 and cleaned up in steps 6-7-8. +The folders created by `*start.sh` scripts are removed by `*drop.sh` scripts. + +## Run test + +### 1. Check if images are correct. + +Run: + +---- + ./1.check.sh +---- + +### 2. Create the database and run all schema scripts. + +This step represents the state that is usually present in the enterprise. +The database is created, and some applications are installed. + +Run: + +---- + ./2.db-start.sh +---- + +### 3. Create Kafka container and topic. + +During this step, the Kafka container is created and started. +Also, the topic for OpenLogReplicator is created. + +Run: + +---- + ./3.kafka-start.sh +---- + +### 4. Create and start Debezium using Logminer. + +This step represents the state when replication is added to the database. + +Run: + +---- + ./4.debezium-start.sh +---- + +After this step, the container with Debezium using Logminer should be up and running. +Transactions are being read from the database and sent to Kafka. +Replication is active. + +### 5. Run SQL and check results + +This step represents a business event when the database is changed and changes are captured by Logminer and published by Debezium connector. + +Run: + +---- + ./5.test.sh +---- + +The expected result is that the Kafka topic contains messages with the JSON representation of the transaction executed by the `sql/test.sql` script. + +### 6. Drop Debezium + +All tests are finished. +Replication can be removed. + +Run: + +---- + ./6.debezium-drop.sh +---- + +### 7. Drop Kafka + +Run: + +---- + ./7.kafka-drop.sh +---- + +After this step, you may go back to step 3 and repeat the process. + +### 8. Drop the database + +Run: + +---- + ./8.db-drop.sh +---- diff --git a/tutorials/Oracle-to-Debezium-logminer/cfg.sh b/tutorials/Oracle-to-Debezium-logminer/cfg.sh new file mode 100755 index 0000000..8bb3fa2 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-logminer/cfg.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . + +export DB_IMAGE=${DB_IMAGE:=oracle/database:21.3.0-xe} +export DB_CONTAINER=${DB_CONTAINER:=ORA1} +export KAFKA_VERSION=3.4.0.Final +export KAFKA_IMAGE=${KAFKA_IMAGE:=quay.io/debezium/kafka:${KAFKA_VERSION}} +export KAFKA_CLUSTER=${KAFKA_CLUSTER:=KAFKA1} +export KAFKA_CONTAINER=${KAFKA_CONTAINER:=KAFKA1} +export KAFKA_BROKER=${KAFKA_BROKER:=kafka:9092} +export KAFKA_TOPIC_LOGMINER=${KAFKA_TOPIC_LOGMINER:=oracle_logminer.USRTBL.ADAM1} +export KAFKA_TOPIC_CONFIGS=${KAFKA_TOPIC_CONFIGS:=topic_configs} +export KAFKA_TOPIC_OFFSETS=${KAFKA_TOPIC_OFFSETS:=topic_offsets} +export KAFKA_TOPIC_STATUSSES=${KAFKA_TOPIC_STATUSSES:=topic_statuses} +export DEBEZIUM_VERSION=3.4.0.Final +export DEBEZIUM_IMAGE=${DEBEZIUM_IMAGE:=quay.io/debezium/connect:${DEBEZIUM_VERSION}} +export DEBEZIUM_CONTAINER=${DEBEZIUM_CONTAINER:=DEBEZIUM1} +export JDBC_FILE=ojdbc8.jar +export JDBC_URL=https://download.oracle.com/otn-pub/otn_software/jdbc/2326/ojdbc8.jar +export INSTANT_CLIENT_FILE=instantclient-basic-linux.x64-23.26.0.0.0.zip +export INSTANT_CLIENT_URL=https://download.oracle.com/otn_software/linux/instantclient/2326000/instantclient-basic-linux.x64-23.26.0.0.0.zip +export INSTANT_CLIENT_DIR=instantclient_23_26 +export DUMP_LOGS=${DUMP_LOGS:=0} + +. ../common/functions.sh diff --git a/tutorials/Oracle-to-Debezium-logminer/compose.yaml b/tutorials/Oracle-to-Debezium-logminer/compose.yaml new file mode 100644 index 0000000..c59628d --- /dev/null +++ b/tutorials/Oracle-to-Debezium-logminer/compose.yaml @@ -0,0 +1,101 @@ +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . + +services: + oracle: + image: "${DB_IMAGE}" + container_name: "${DB_CONTAINER}" + volumes: + - ./fra:/opt/oracle/fra + - ./oradata:/opt/oracle/oradata + - ./sql:/opt/sql + - ./setup:/opt/oracle/scripts/setup + networks: + - isolated_network + environment: + - ORACLE_CHARACTERSET=AL32UTF8 + build: + context: . + shm_size: '2gb' + shm_size: '2gb' + + kafka: + image: "${KAFKA_IMAGE}" + container_name: "${KAFKA_CONTAINER}" + profiles: ["kafka"] + volumes: + - ./kafka:/var/lib/kafka/data + networks: + - isolated_network + environment: + # KRaft parameters + - CLUSTER_ID=${KAFKA_CLUSTER} + - KAFKA_NODE_ID=1 + - KAFKA_PROCESS_ROLES=broker,controller + - KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS=kafka:9093 + # Listeners + - KAFKA_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093 + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092 + - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT + # Performance & configuration + - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1 + - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1 + healthcheck: + test: ["CMD", "/kafka/bin/kafka-topics.sh", "--bootstrap-server", "${KAFKA_BROKER}", "--list"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 60s + + connect: + image: "${DEBEZIUM_IMAGE}" + container_name: "${DEBEZIUM_CONTAINER}" + profiles: ["debezium"] + depends_on: + - kafka + environment: + - BOOTSTRAP_SERVERS=kafka:9092 + - GROUP_ID=1 + - CONFIG_STORAGE_TOPIC=${KAFKA_TOPIC_CONFIGS} + - OFFSET_STORAGE_TOPIC=${KAFKA_TOPIC_OFFSETS} + - STATUS_STORAGE_TOPIC=${KAFKA_TOPIC_STATUSSES} + # Debezium specific settings + - CONFIG_STORAGE_REPLICATION_FACTOR=1 + - OFFSET_STORAGE_REPLICATION_FACTOR=1 + - STATUS_STORAGE_REPLICATION_FACTOR=1 + # Oracle connector specific settings + - LD_LIBRARY_PATH=/instant_client + volumes: + - ./oracle_libs/${JDBC_FILE}:/kafka/connect/debezium-connector-oracle/${JDBC_FILE} + - ./${INSTANT_CLIENT_DIR}:/instant_client + - ~/debezium/debezium-connector-oracle-3.4.0.Final.jar:/kafka/connect/debezium-connector-oracle/debezium-connector-oracle-3.4.0.Final.jar + networks: + - isolated_network + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:8083/" ] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + +networks: + isolated_network: + internal: true diff --git a/tutorials/Oracle-to-Debezium-logminer/setup/config.sql b/tutorials/Oracle-to-Debezium-logminer/setup/config.sql new file mode 100644 index 0000000..780a38d --- /dev/null +++ b/tutorials/Oracle-to-Debezium-logminer/setup/config.sql @@ -0,0 +1,10 @@ +SHUTDOWN IMMEDIATE; +STARTUP MOUNT; +ALTER DATABASE ARCHIVELOG; +ALTER DATABASE OPEN; +ALTER DATABASE ADD SUPPLEMENTAL LOG DATA; +ALTER SYSTEM ARCHIVE LOG CURRENT; + +ALTER SYSTEM SET DB_CREATE_FILE_DEST = '/opt/oracle/oradata' SCOPE = BOTH; +ALTER SYSTEM SET DB_RECOVERY_FILE_DEST_SIZE = '250G' SCOPE = BOTH; +ALTER SYSTEM SET DB_RECOVERY_FILE_DEST = '/opt/oracle/fra' SCOPE = BOTH; diff --git a/tutorials/Oracle-to-Debezium-logminer/sql/drop-usrdbz_logminer.sql b/tutorials/Oracle-to-Debezium-logminer/sql/drop-usrdbz_logminer.sql new file mode 100644 index 0000000..b260dff --- /dev/null +++ b/tutorials/Oracle-to-Debezium-logminer/sql/drop-usrdbz_logminer.sql @@ -0,0 +1,7 @@ +ALTER SESSION SET CONTAINER = CDB$ROOT; +DROP USER C##USRDBZ_LOGMINER CASCADE; + +ALTER SESSION SET CONTAINER = XEPDB1; +DROP TABLESPACE DBZTBLSPCLOGMINER; +ALTER SESSION SET CONTAINER = CDB$ROOT; +DROP TABLESPACE DBZTBLSPCLOGMINER; diff --git a/tutorials/Oracle-to-Debezium-logminer/sql/schema-usrdbz_logminer.sql b/tutorials/Oracle-to-Debezium-logminer/sql/schema-usrdbz_logminer.sql new file mode 100644 index 0000000..a08f816 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-logminer/sql/schema-usrdbz_logminer.sql @@ -0,0 +1,34 @@ +CREATE TABLESPACE DBZTBLSPCLOGMINER DATAFILE SIZE 25M AUTOEXTEND ON MAXSIZE UNLIMITED; + +ALTER SESSION SET CONTAINER = CDB$ROOT; +CREATE TABLESPACE DBZTBLSPCLOGMINER DATAFILE SIZE 25M AUTOEXTEND ON MAXSIZE UNLIMITED; + +ALTER SESSION SET CONTAINER = CDB$ROOT; +CREATE USER C##USRDBZ_LOGMINER IDENTIFIED BY dbzpwd + DEFAULT TABLESPACE DBZTBLSPCLOGMINER + QUOTA UNLIMITED ON DBZTBLSPCLOGMINER + CONTAINER = ALL; + +ALTER SESSION SET CONTAINER = CDB$ROOT; +GRANT CREATE SESSION TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT CREATE TABLE TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT CREATE SEQUENCE TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT SET CONTAINER TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT SELECT_CATALOG_ROLE TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT EXECUTE_CATALOG_ROLE TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT SELECT ANY TRANSACTION TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT SELECT ANY DICTIONARY TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT SELECT ANY TABLE TO C##USRDBZ_LOGMINER CONTAINER=ALL; + +GRANT LOGMINING TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT EXECUTE ON DBMS_LOGMNR TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT EXECUTE ON DBMS_LOGMNR_D TO C##USRDBZ_LOGMINER CONTAINER=ALL; + +GRANT SELECT ON V_$LOG TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT SELECT ON V_$LOGFILE TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT SELECT ON V_$LOGMNR_CONTENTS TO C##USRDBZ_LOGMINER CONTAINER=ALL; +GRANT SELECT ON V_$ARCHIVED_LOG TO C##USRDBZ_LOGMINER CONTAINER=ALL; + +ALTER SESSION SET CONTAINER = XEPDB1; +GRANT FLASHBACK ON USRTBL.ADAM1 TO C##USRDBZ_LOGMINER; +GRANT SELECT ON USRTBL.ADAM1 TO C##USRDBZ_LOGMINER; diff --git a/tutorials/Oracle-to-Debezium-logminer/sql/schema-usrtbl.sql b/tutorials/Oracle-to-Debezium-logminer/sql/schema-usrtbl.sql new file mode 100644 index 0000000..40ca62f --- /dev/null +++ b/tutorials/Oracle-to-Debezium-logminer/sql/schema-usrtbl.sql @@ -0,0 +1,24 @@ +ALTER SESSION SET CONTAINER = CDB$ROOT; +ALTER SYSTEM ARCHIVE LOG CURRENT; + +ALTER SESSION SET CONTAINER = XEPDB1; +CREATE TABLESPACE TBLS1 DATAFILE SIZE 100M AUTOEXTEND ON NEXT 100M; +ALTER TABLESPACE TBLS1 FORCE LOGGING; + +CREATE USER USRTBL IDENTIFIED BY USRTBLPWD DEFAULT TABLESPACE TBLS1 TEMPORARY TABLESPACE TEMP; +ALTER USER USRTBL QUOTA UNLIMITED ON TBLS1; + +GRANT CONNECT TO USRTBL; +GRANT RESOURCE TO USRTBL; + +CREATE TABLE USRTBL.ADAM1( + ID NUMBER NOT NULL, + NAME VARCHAR2(30), + COUNT NUMBER, + START_TIME TIMESTAMP +); +ALTER TABLE USRTBL.ADAM1 ADD CONSTRAINT ADAM1PK PRIMARY KEY(ID); +ALTER TABLE USRTBL.ADAM1 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER SESSION SET CONTAINER = CDB$ROOT; +ALTER SYSTEM ARCHIVE LOG CURRENT; diff --git a/tutorials/Oracle-to-Debezium-logminer/sql/test.sql b/tutorials/Oracle-to-Debezium-logminer/sql/test.sql new file mode 100644 index 0000000..4037e50 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-logminer/sql/test.sql @@ -0,0 +1,6 @@ +INSERT INTO USRTBL.ADAM1 VALUES (1, 'Adam Leszczyński', 10, TO_DATE('2019-08-01 12:34:56', 'YYYY-MM-DD HH24:MI:SS')); +COMMIT; +UPDATE USRTBL.ADAM1 SET COUNT = COUNT + 1; +COMMIT; +DELETE FROM USRTBL.ADAM1; +COMMIT; diff --git a/tutorials/Oracle-to-Debezium-olr/1.check.sh b/tutorials/Oracle-to-Debezium-olr/1.check.sh new file mode 100755 index 0000000..93e8fd1 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-olr/1.check.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "1. checking docker images" +docker_check "${DB_IMAGE}" +docker_check "${OLR_IMAGE}" +finish diff --git a/tutorials/Oracle-to-Debezium-olr/10.db-drop.sh b/tutorials/Oracle-to-Debezium-olr/10.db-drop.sh new file mode 100755 index 0000000..1c83cd1 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-olr/10.db-drop.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "10. dropping database container" +docker_rm ${DB_CONTAINER} +docker_down +file_rm fra oradata oracle_libs sql/schema-usrtbl.out "${INSTANT_CLIENT_DIR}" +finish diff --git a/tutorials/Oracle-to-Debezium-olr/2.db-start.sh b/tutorials/Oracle-to-Debezium-olr/2.db-start.sh new file mode 100755 index 0000000..dceb923 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-olr/2.db-start.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "2. creating and starting database container" +db_files "oracle_libs" +docker_up +db_wait "${DB_CONTAINER}" +db_sql "${DB_CONTAINER}" /opt/sql/schema-usrtbl.sql /opt/sql/schema-usrtbl.out +db_client +finish diff --git a/tutorials/Oracle-to-Debezium-olr/3.kafka-start.sh b/tutorials/Oracle-to-Debezium-olr/3.kafka-start.sh new file mode 100755 index 0000000..a333e78 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-olr/3.kafka-start.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "3. creating and starting Kafka container" +kafka_files +docker_up_wait --profile kafka +kafka_create_topic_compact "${KAFKA_CONTAINER}" "${KAFKA_BROKER}" "${KAFKA_TOPIC_CONFIGS}" +kafka_create_topic_compact "${KAFKA_CONTAINER}" "${KAFKA_BROKER}" "${KAFKA_TOPIC_OFFSETS}" +kafka_create_topic_compact "${KAFKA_CONTAINER}" "${KAFKA_BROKER}" "${KAFKA_TOPIC_STATUSSES}" +kafka_list_topics "${KAFKA_CONTAINER}" "${KAFKA_BROKER}" +finish diff --git a/tutorials/Oracle-to-Debezium-olr/4.olr-start.sh b/tutorials/Oracle-to-Debezium-olr/4.olr-start.sh new file mode 100755 index 0000000..2182f2e --- /dev/null +++ b/tutorials/Oracle-to-Debezium-olr/4.olr-start.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "4. creating and starting OpenLogReplicator container" +olr_files +db_sql "${DB_CONTAINER}" /opt/sql/schema-usrolr.sql /opt/sql/schema-usrolr.out +docker_up --profile openlogreplicator --profile kafka +# olr_wait_for_start "${OLR_CONTAINER}" "ready" +sleep 10 +finish diff --git a/tutorials/Oracle-to-Debezium-olr/5.debezium-start.sh b/tutorials/Oracle-to-Debezium-olr/5.debezium-start.sh new file mode 100755 index 0000000..dc8a51e --- /dev/null +++ b/tutorials/Oracle-to-Debezium-olr/5.debezium-start.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "5. creating and starting Debezium container" +db_sql "${DB_CONTAINER}" /opt/sql/schema-usrdbz_olr.sql /opt/sql/schema-usrdbz_olr.out +docker_up_wait --profile debezium --profile kafka +debezium_olr "${DEBEZIUM_CONTAINER}" "${KAFKA_BROKER}" +debezium_wait_for_connect "${DEBEZIUM_CONTAINER}" "oracle-olr-connector" +finish diff --git a/tutorials/Oracle-to-Debezium-olr/6.test.sh b/tutorials/Oracle-to-Debezium-olr/6.test.sh new file mode 100755 index 0000000..82ba332 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-olr/6.test.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "6a. running test - verify data in Kafka topic" +db_sql "${DB_CONTAINER}" /opt/sql/test.sql /opt/sql/test.out +kafka_wait_for_messages "${KAFKA_CONTAINER}" "${KAFKA_BROKER}" "${KAFKA_TOPIC_OLR}" "COUNT" 3 + +echo "6. running test - restart Debezium" +docker stop ${DEBEZIUM_CONTAINER} +docker start ${DEBEZIUM_CONTAINER} +debezium_wait_for_connect "${DEBEZIUM_CONTAINER}" "oracle-olr-connector" +db_sql "${DB_CONTAINER}" /opt/sql/test.sql /opt/sql/test.out +kafka_wait_for_messages "${KAFKA_CONTAINER}" "${KAFKA_BROKER}" "${KAFKA_TOPIC_OLR}" "COUNT" 6 + +echo "6. running test - restart database" +docker stop ${DB_CONTAINER} +docker start ${DB_CONTAINER} +db_wait "${DB_CONTAINER}" +sleep 30 +db_sql "${DB_CONTAINER}" /opt/sql/test.sql /opt/sql/test.out +kafka_wait_for_messages "${KAFKA_CONTAINER}" "${KAFKA_BROKER}" "${KAFKA_TOPIC_OLR}" "COUNT" 9 + +#echo "6. running test - restart OLR" +#docker stop ${OLR_CONTAINER} +#docker start ${OLR_CONTAINER} +#sleep 10 +#db_sql "${DB_CONTAINER}" /opt/sql/test.sql /opt/sql/test.out +#kafka_wait_for_messages "${KAFKA_CONTAINER}" "${KAFKA_BROKER}" "${KAFKA_TOPIC_OLR}" "COUNT" 12 + +finish diff --git a/tutorials/Oracle-to-Debezium-olr/7.debezium-drop.sh b/tutorials/Oracle-to-Debezium-olr/7.debezium-drop.sh new file mode 100755 index 0000000..c30433f --- /dev/null +++ b/tutorials/Oracle-to-Debezium-olr/7.debezium-drop.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "7. dropping Debezium container" +docker_rm "${DEBEZIUM_CONTAINER}" +db_sql "${DB_CONTAINER}" /opt/sql/drop-usrdbz_olr.sql /opt/sql/drop-usrdbz_olr.out +file_rm sql/schema-usrdbz_olr.out sql/drop-usrdbz_olr.out +finish diff --git a/tutorials/Oracle-to-Debezium-olr/8.olr-drop.sh b/tutorials/Oracle-to-Debezium-olr/8.olr-drop.sh new file mode 100755 index 0000000..80a3bc1 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-olr/8.olr-drop.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "8. dropping OpenLogReplicator container" +docker_rm "${OLR_CONTAINER}" +db_sql "${DB_CONTAINER}" /opt/sql/drop-usrolr.sql /opt/sql/drop-usrolr.out +file_rm sql/drop-usrolr.out sql/schema-usrolr.out sql/test.out checkpoint log +finish diff --git a/tutorials/Oracle-to-Debezium-olr/9.kafka-drop.sh b/tutorials/Oracle-to-Debezium-olr/9.kafka-drop.sh new file mode 100755 index 0000000..efd5903 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-olr/9.kafka-drop.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . +set -e + +. cfg.sh +. ../common/functions.sh + +echo "9. dropping Kafka container" +docker_rm "${KAFKA_CONTAINER}" +file_rm kafka +finish diff --git a/tutorials/Oracle-to-Debezium-olr/README.md b/tutorials/Oracle-to-Debezium-olr/README.md new file mode 100644 index 0000000..b65db31 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-olr/README.md @@ -0,0 +1,116 @@ +# Replicating Oracle transactions to Kafka with Debezium + +Goal: reference replication with Debezium by capturing transactional changes from a database and delivering them to a Kafka topic, covering environment setup, enabling replication, executing a test change that produces the output, and cleanup. +Uses OpenLogReplicator as the capture method. + +## Create Docker images + +The following images are required for this tutorial: +- `oracle/database:21.3.0-xe` - Oracle Database 21.3 XE +- `quay.io/debezium/kafka` - Kafka +- `quay.io/debezium/connect` - Debezium connector + +Refer to [Docker images](../../images/README.md) for image creation scripts. + +## Container names + +The default container names are: +- `ORA1` - Oracle Database 21.3 XE +- `KAFKA1` - Kafka broker +- `DEBEZIUM1` - Debezium connector + +For ease of use, a Zookeeper container is not created separately, since a newer Kafka version is used which relies on KRaft. +Container names are defined in the `cfg.sh` script. + +## Folders used by this tutorial + +All files for the database devices and redo log files are placed outside the containers. +The folders are created in steps 2-3-4 and cleaned up in steps 6-7-8. +The folders created by `*start.sh` scripts are removed by `*drop.sh` scripts. + +## Run test + +### 1. Check if images are correct. + +Run: + +---- + ./1.check.sh +---- + +### 2. Create the database and run all schema scripts. + +This step represents the state that is usually present in the enterprise. +The database is created, and some applications are installed. + +Run: + +---- + ./2.db-start.sh +---- + +### 3. Create Kafka container and topic. + +During this step, the Kafka container is created and started. +Also, the topic for OpenLogReplicator is created. + +Run: + +---- + ./3.kafka-start.sh +---- + +### 4. Create and start Debezium using OpenLogReplicator. + +This step represents the state when replication is added to the database. + +Run: + +---- + ./4.debezium-start.sh +---- + +After this step, the container with Debezium using OpenLogReplicator should be up and running. +Transactions are being read from the database and sent to Kafka. +Replication is active. + +### 5. Run SQL and check results + +This step represents a business event when the database is changed and changes are captured by OpenLogReplicator and published by Debezium connector. + +Run: + +---- + ./5.test.sh +---- + +The expected result is that the Kafka topic contains messages with the JSON representation of the transaction executed by the `sql/test.sql` script. + +### 6. Drop Debezium + +All tests are finished. +Replication can be removed. + +Run: + +---- + ./6.debezium-drop.sh +---- + +### 7. Drop Kafka + +Run: + +---- + ./7.kafka-drop.sh +---- + +After this step, you may go back to step 3 and repeat the process. + +### 8. Drop the database + +Run: + +---- + ./8.db-drop.sh +---- diff --git a/tutorials/Oracle-to-Debezium-olr/cfg.sh b/tutorials/Oracle-to-Debezium-olr/cfg.sh new file mode 100755 index 0000000..55a419d --- /dev/null +++ b/tutorials/Oracle-to-Debezium-olr/cfg.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . + +export DB_IMAGE=${DB_IMAGE:=oracle/database:21.3.0-xe} +export DB_CONTAINER=${DB_CONTAINER:=ORA1} +export KAFKA_VERSION=3.4.0.Final +export KAFKA_IMAGE=${KAFKA_IMAGE:=quay.io/debezium/kafka:${KAFKA_VERSION}} +export KAFKA_CLUSTER=${KAFKA_CLUSTER:=KAFKA1} +export KAFKA_CONTAINER=${KAFKA_CONTAINER:=KAFKA1} +export KAFKA_BROKER=${KAFKA_BROKER:=kafka:9092} +export KAFKA_TOPIC_OLR=${KAFKA_TOPIC_OLR:=oracle_olr.USRTBL.ADAM1} +export KAFKA_TOPIC_CONFIGS=${KAFKA_TOPIC_CONFIGS:=topic_configs} +export KAFKA_TOPIC_OFFSETS=${KAFKA_TOPIC_OFFSETS:=topic_offsets} +export KAFKA_TOPIC_STATUSSES=${KAFKA_TOPIC_STATUSSES:=topic_statuses} +export OLR_IMAGE=${OLR_IMAGE:=bersler/openlogreplicator:tutorial} +export OLR_CONTAINER=${OLR_CONTAINER:=OLR1} +export OLR_CUSTOM=${OLR_CUSTOM:=} +export DEBEZIUM_VERSION=3.4.0.Final +export DEBEZIUM_IMAGE=${DEBEZIUM_IMAGE:=quay.io/debezium/connect:${DEBEZIUM_VERSION}} +export DEBEZIUM_CONTAINER=${DEBEZIUM_CONTAINER:=DEBEZIUM1} +export JDBC_FILE=ojdbc8.jar +export JDBC_URL=https://download.oracle.com/otn-pub/otn_software/jdbc/2326/ojdbc8.jar +export INSTANT_CLIENT_FILE=instantclient-basic-linux.x64-23.26.0.0.0.zip +export INSTANT_CLIENT_URL=https://download.oracle.com/otn_software/linux/instantclient/2326000/instantclient-basic-linux.x64-23.26.0.0.0.zip +export INSTANT_CLIENT_DIR=instantclient_23_26 +export DUMP_LOGS=${DUMP_LOGS:=0} diff --git a/tutorials/Oracle-to-Debezium-olr/compose.yaml b/tutorials/Oracle-to-Debezium-olr/compose.yaml new file mode 100644 index 0000000..231537b --- /dev/null +++ b/tutorials/Oracle-to-Debezium-olr/compose.yaml @@ -0,0 +1,120 @@ +# Copyright (C) 2018-2026 Adam Leszczynski (aleszczynski@bersler.com) +# +# This file is part of OpenLogReplicator-tutorials +# +# Open Log Replicator is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# Open Log Replicator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Open Log Replicator; see the file LICENSE.txt If not see +# . + +services: + oracle: + image: "${DB_IMAGE}" + container_name: "${DB_CONTAINER}" + volumes: + - ./fra:/opt/oracle/fra + - ./oradata:/opt/oracle/oradata + - ./sql:/opt/sql + - ./setup:/opt/oracle/scripts/setup + networks: + - isolated_network + environment: + - ORACLE_CHARACTERSET=AL32UTF8 + build: + context: . + shm_size: '2gb' + shm_size: '2gb' + + kafka: + image: "${KAFKA_IMAGE}" + container_name: "${KAFKA_CONTAINER}" + profiles: ["kafka"] + volumes: + - ./kafka:/var/lib/kafka/data + networks: + - isolated_network + environment: + # KRaft parameters + - CLUSTER_ID=${KAFKA_CLUSTER} + - KAFKA_NODE_ID=1 + - KAFKA_PROCESS_ROLES=broker,controller + - KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS=kafka:9093 + # Listeners + - KAFKA_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093 + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092 + - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT + # Performance & configuration + - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1 + - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1 + healthcheck: + test: ["CMD", "/kafka/bin/kafka-topics.sh", "--bootstrap-server", "${KAFKA_BROKER}", "--list"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 60s + + openlogreplicator: + image: "${OLR_IMAGE}" + container_name: "${OLR_CONTAINER}" + profiles: ["openlogreplicator"] + depends_on: + - oracle + - kafka + volumes: + - ./bin:/opt/bin + - ./checkpoint:/opt/OpenLogReplicator/checkpoint + - ./dump:/opt/dump + - ./fra:/opt/fra + - ./log:/opt/OpenLogReplicator/log + - ./oradata:/opt/oradata + - ./scripts:/opt/OpenLogReplicator/scripts + restart: "no" + networks: + - isolated_network + + connect: + image: "${DEBEZIUM_IMAGE}" + container_name: "${DEBEZIUM_CONTAINER}" + profiles: ["debezium"] + depends_on: + - kafka + environment: + - BOOTSTRAP_SERVERS=kafka:9092 + - GROUP_ID=1 + - CONFIG_STORAGE_TOPIC=${KAFKA_TOPIC_CONFIGS} + - OFFSET_STORAGE_TOPIC=${KAFKA_TOPIC_OFFSETS} + - STATUS_STORAGE_TOPIC=${KAFKA_TOPIC_STATUSSES} + # Debezium specific settings + - CONFIG_STORAGE_REPLICATION_FACTOR=1 + - OFFSET_STORAGE_REPLICATION_FACTOR=1 + - STATUS_STORAGE_REPLICATION_FACTOR=1 + # Oracle connector specific settings + - LD_LIBRARY_PATH=/instant_client + volumes: + - ./oracle_libs/${JDBC_FILE}:/kafka/connect/debezium-connector-oracle/${JDBC_FILE} + - ./${INSTANT_CLIENT_DIR}:/instant_client + - ~/debezium/debezium-connector-oracle-3.4.0.Final.jar:/kafka/connect/debezium-connector-oracle/debezium-connector-oracle-3.4.0.Final.jar + networks: + - isolated_network + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:8083/" ] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + +networks: + isolated_network: + internal: true diff --git a/tutorials/Oracle-to-Debezium-olr/scripts/OpenLogReplicator.json b/tutorials/Oracle-to-Debezium-olr/scripts/OpenLogReplicator.json new file mode 100644 index 0000000..d07a696 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-olr/scripts/OpenLogReplicator.json @@ -0,0 +1,43 @@ +{ + "version": "2.0.0", + "log-level": 3, + "trace": 524288, + "dump-redo-log": 2, + "dump-raw-data": 1, + "dump-path": "/opt/dump", + "metrics": { + "type": "prometheus", + "bind": "0.0.0.0:8080" + }, + "source": [ + { + "alias": "S1", + "name": "ORA1", + "reader": { + "type": "online", + "path-mapping": ["/opt/oracle/oradata", "/opt/oradata", "/opt/oracle/fra", "/opt/fra"], + "user": "USROLR", + "password": "USROLRPWD", + "server": "//oracle:1521/XEPDB1" + }, + "format": { + "type": "debezium" + }, + "filter": { + "table": [ + {"owner": "USRTBL", "table": "ADAM.*"} + ] + } + } + ], + "target": [ + { + "alias": "K1", + "source": "S1", + "writer": { + "type": "network", + "uri": "0.0.0.0:5000" + } + } + ] +} diff --git a/tutorials/Oracle-to-Debezium-olr/setup/config.sql b/tutorials/Oracle-to-Debezium-olr/setup/config.sql new file mode 100644 index 0000000..780a38d --- /dev/null +++ b/tutorials/Oracle-to-Debezium-olr/setup/config.sql @@ -0,0 +1,10 @@ +SHUTDOWN IMMEDIATE; +STARTUP MOUNT; +ALTER DATABASE ARCHIVELOG; +ALTER DATABASE OPEN; +ALTER DATABASE ADD SUPPLEMENTAL LOG DATA; +ALTER SYSTEM ARCHIVE LOG CURRENT; + +ALTER SYSTEM SET DB_CREATE_FILE_DEST = '/opt/oracle/oradata' SCOPE = BOTH; +ALTER SYSTEM SET DB_RECOVERY_FILE_DEST_SIZE = '250G' SCOPE = BOTH; +ALTER SYSTEM SET DB_RECOVERY_FILE_DEST = '/opt/oracle/fra' SCOPE = BOTH; diff --git a/tutorials/Oracle-to-Debezium-olr/sql/drop-usrdbz_olr.sql b/tutorials/Oracle-to-Debezium-olr/sql/drop-usrdbz_olr.sql new file mode 100644 index 0000000..3d93c70 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-olr/sql/drop-usrdbz_olr.sql @@ -0,0 +1,7 @@ +ALTER SESSION SET CONTAINER = CDB$ROOT; +DROP USER C##USRDBZ_OLR CASCADE; + +ALTER SESSION SET CONTAINER = XEPDB1; +DROP TABLESPACE DBZTBLSPCOLR; +ALTER SESSION SET CONTAINER = CDB$ROOT; +DROP TABLESPACE DBZTBLSPCOLR; diff --git a/tutorials/Oracle-to-Debezium-olr/sql/drop-usrolr.sql b/tutorials/Oracle-to-Debezium-olr/sql/drop-usrolr.sql new file mode 100644 index 0000000..5c61315 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-olr/sql/drop-usrolr.sql @@ -0,0 +1 @@ +DROP USER USROLR CASCADE; diff --git a/tutorials/Oracle-to-Debezium-olr/sql/schema-usrdbz_olr.sql b/tutorials/Oracle-to-Debezium-olr/sql/schema-usrdbz_olr.sql new file mode 100644 index 0000000..98ba3ee --- /dev/null +++ b/tutorials/Oracle-to-Debezium-olr/sql/schema-usrdbz_olr.sql @@ -0,0 +1,30 @@ +CREATE TABLESPACE DBZTBLSPCOLR DATAFILE SIZE 25M AUTOEXTEND ON MAXSIZE UNLIMITED; + +ALTER SESSION SET CONTAINER = CDB$ROOT; +CREATE TABLESPACE DBZTBLSPCOLR DATAFILE SIZE 25M AUTOEXTEND ON MAXSIZE UNLIMITED; + +ALTER SESSION SET CONTAINER = CDB$ROOT; +CREATE USER C##USRDBZ_OLR IDENTIFIED BY dbzpwd + DEFAULT TABLESPACE DBZTBLSPCOLR + QUOTA UNLIMITED ON DBZTBLSPCOLR + CONTAINER = ALL; + +ALTER SESSION SET CONTAINER = CDB$ROOT; +GRANT CREATE SESSION TO C##USRDBZ_OLR CONTAINER=ALL; +GRANT CREATE TABLE TO C##USRDBZ_OLR CONTAINER=ALL; +GRANT CREATE SEQUENCE TO C##USRDBZ_OLR CONTAINER=ALL; +GRANT SET CONTAINER TO C##USRDBZ_OLR CONTAINER=ALL; +GRANT SELECT_CATALOG_ROLE TO C##USRDBZ_OLR CONTAINER=ALL; +GRANT EXECUTE_CATALOG_ROLE TO C##USRDBZ_OLR CONTAINER=ALL; +GRANT SELECT ANY TRANSACTION TO C##USRDBZ_OLR CONTAINER=ALL; +GRANT SELECT ANY DICTIONARY TO C##USRDBZ_OLR CONTAINER=ALL; +GRANT SELECT ANY TABLE TO C##USRDBZ_OLR CONTAINER=ALL; + +GRANT SELECT ON V_$LOG TO C##USRDBZ_OLR CONTAINER=ALL; +GRANT SELECT ON V_$LOGFILE TO C##USRDBZ_OLR CONTAINER=ALL; +GRANT SELECT ON V_$LOGMNR_CONTENTS TO C##USRDBZ_OLR CONTAINER=ALL; +GRANT SELECT ON V_$ARCHIVED_LOG TO C##USRDBZ_OLR CONTAINER=ALL; + +ALTER SESSION SET CONTAINER = XEPDB1; +GRANT FLASHBACK ON USRTBL.ADAM1 TO C##USRDBZ_OLR; +GRANT SELECT ON USRTBL.ADAM1 TO C##USRDBZ_OLR; diff --git a/tutorials/Oracle-to-Debezium-olr/sql/schema-usrolr.sql b/tutorials/Oracle-to-Debezium-olr/sql/schema-usrolr.sql new file mode 100644 index 0000000..c1bc4d9 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-olr/sql/schema-usrolr.sql @@ -0,0 +1,46 @@ +CREATE USER USROLR IDENTIFIED BY USROLRPWD TEMPORARY TABLESPACE TEMP; + +GRANT CONNECT TO USROLR; +GRANT RESOURCE TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.CCOL$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.CDEF$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.COL$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.DEFERRED_STG$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.ECOL$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.LOB$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.LOBCOMPPART$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.LOBFRAG$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.OBJ$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.TAB$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.TABCOMPART$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.TABPART$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.TABSUBPART$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.TS$ TO USROLR; +GRANT SELECT, FLASHBACK ON SYS.USER$ TO USROLR; +GRANT SELECT ON SYS.V_$ARCHIVED_LOG TO USROLR; +GRANT SELECT ON SYS.V_$DATABASE TO USROLR; +GRANT SELECT ON SYS.V_$DATABASE_INCARNATION TO USROLR; +GRANT SELECT ON SYS.V_$LOG TO USROLR; +GRANT SELECT ON SYS.V_$LOGFILE TO USROLR; +GRANT SELECT ON SYS.V_$PARAMETER TO USROLR; +GRANT SELECT ON SYS.V_$PDBS TO USROLR; +GRANT SELECT ON SYS.V_$STANDBY_LOG TO USROLR; +GRANT SELECT ON SYS.V_$TRANSPORTABLE_PLATFORM TO USROLR; +GRANT SELECT, FLASHBACK ON XDB.XDB$TTSET TO USROLR; +DECLARE +CURSOR C1 IS SELECT TOKSUF FROM XDB.XDB$TTSET; +CMD VARCHAR2(2000); +BEGIN +FOR C IN C1 LOOP + CMD := 'GRANT SELECT, FLASHBACK ON XDB.X$NM' || C.TOKSUF || ' TO USROLR'; +EXECUTE IMMEDIATE CMD; +CMD := 'GRANT SELECT, FLASHBACK ON XDB.X$QN' || C.TOKSUF || ' TO USROLR'; +EXECUTE IMMEDIATE CMD; +CMD := 'GRANT SELECT, FLASHBACK ON XDB.X$PT' || C.TOKSUF || ' TO USROLR'; +EXECUTE IMMEDIATE CMD; +END LOOP; +END; +/ + +ALTER SESSION SET CONTAINER = CDB$ROOT; +ALTER SYSTEM ARCHIVE LOG CURRENT; diff --git a/tutorials/Oracle-to-Debezium-olr/sql/schema-usrtbl.sql b/tutorials/Oracle-to-Debezium-olr/sql/schema-usrtbl.sql new file mode 100644 index 0000000..40ca62f --- /dev/null +++ b/tutorials/Oracle-to-Debezium-olr/sql/schema-usrtbl.sql @@ -0,0 +1,24 @@ +ALTER SESSION SET CONTAINER = CDB$ROOT; +ALTER SYSTEM ARCHIVE LOG CURRENT; + +ALTER SESSION SET CONTAINER = XEPDB1; +CREATE TABLESPACE TBLS1 DATAFILE SIZE 100M AUTOEXTEND ON NEXT 100M; +ALTER TABLESPACE TBLS1 FORCE LOGGING; + +CREATE USER USRTBL IDENTIFIED BY USRTBLPWD DEFAULT TABLESPACE TBLS1 TEMPORARY TABLESPACE TEMP; +ALTER USER USRTBL QUOTA UNLIMITED ON TBLS1; + +GRANT CONNECT TO USRTBL; +GRANT RESOURCE TO USRTBL; + +CREATE TABLE USRTBL.ADAM1( + ID NUMBER NOT NULL, + NAME VARCHAR2(30), + COUNT NUMBER, + START_TIME TIMESTAMP +); +ALTER TABLE USRTBL.ADAM1 ADD CONSTRAINT ADAM1PK PRIMARY KEY(ID); +ALTER TABLE USRTBL.ADAM1 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +ALTER SESSION SET CONTAINER = CDB$ROOT; +ALTER SYSTEM ARCHIVE LOG CURRENT; diff --git a/tutorials/Oracle-to-Debezium-olr/sql/test.sql b/tutorials/Oracle-to-Debezium-olr/sql/test.sql new file mode 100644 index 0000000..4037e50 --- /dev/null +++ b/tutorials/Oracle-to-Debezium-olr/sql/test.sql @@ -0,0 +1,6 @@ +INSERT INTO USRTBL.ADAM1 VALUES (1, 'Adam Leszczyński', 10, TO_DATE('2019-08-01 12:34:56', 'YYYY-MM-DD HH24:MI:SS')); +COMMIT; +UPDATE USRTBL.ADAM1 SET COUNT = COUNT + 1; +COMMIT; +DELETE FROM USRTBL.ADAM1; +COMMIT;