feat: rdkafka 2.3, upgrade deps (#2)

* feat: rdkafka 2.3, upgrade deps
* chore: add node 21
* chore: use custom pnpm store location
* chore: examples to build arm based kafka

Author: AVVS

.husky/commit-msg

@@ -1,4 +1,4 @@
 #!/bin/sh
 . "$(dirname "$0")/_/husky.sh"
 
-"`$(npm bin)/mdep bin commitlint`" --edit $1
+"`npm x -- mdep bin commitlint`" --edit $1

.semaphore/semaphore.yml

@@ -8,16 +8,16 @@ agent:
 global_job_config:
   prologue:
     commands:
-      - sem-version node 18
-      - curl -fsSL https://get.pnpm.io/install.sh | env PNPM_VERSION=7.25.1 sh -
+      - sem-version node 20
+      - curl -fsSL https://get.pnpm.io/install.sh | env PNPM_VERSION=8.10.2 sh -
       - source /home/semaphore/.bashrc
       - pnpm config set store-dir=~/.pnpm-store
       - checkout
       - git submodule init
       - git submodule update
       - cache restore node-$(checksum pnpm-lock.yaml)
       - pnpm i --frozen-lockfile --prefer-offline --ignore-scripts
-      - cache store node-$(checksum pnpm-lock.yaml) ~/.pnpm-store
+      - cache store node-$(checksum pnpm-lock.yaml) $(pnpm config get store-dir)
   env_vars:
   - name: BUILD_LIBRDKAFKA
     value: '0'
@@ -45,14 +45,8 @@ blocks:
       - name: pre-build & publish binaries
         matrix:
         - env_var: NODE_VER
-          values: ["16", "18", "19"]
+          values: ["18", "20", "21"]
         - env_var: platform
           values: ["-rdkafka", "-debian-rdkafka"]
         commands:
-          - cp ~/.env.aws-s3-credentials .env
-          - env IMAGE_TAG=${NODE_VER}${platform} UID=${UID} PNPM_STORE=$(pnpm config get store-dir) docker-compose up -d
-          - docker-compose exec tester pnpm i --frozen-lockfile --prefer-offline --ignore-scripts
-          - docker-compose exec tester pnpm binary:build
-          - docker-compose exec tester pnpm binary:package
-          - docker-compose exec tester pnpm binary:test
-          - docker-compose exec tester pnpm binary:publish
+          - ./ci/build-and-publish.sh

CONTRIBUTING.md

@@ -207,18 +207,18 @@ Steps to update:
     git checkout 063a9ae7a65cebdf1cc128da9815c05f91a2a996 # for version 1.8.2
     ```
 
+    If you get an error during that checkout command, double check that the submodule was initialized / cloned! You may need to run `git submodule update --init --recursive`
+
 1. Update [`config.d.ts`](https://github.com/Blizzard/node-rdkafka/blob/master/config.d.ts) and [`errors.d.ts`](https://github.com/Blizzard/node-rdkafka/blob/master/errors.d.ts) TypeScript definitions by running:
     ```bash
     node ci/librdkafka-defs-generator.js
     ```
     Note: This is ran automatically during CI flows but it's good to run it during the version upgrade pull request.
 
-1. Run `npm install` to build with the new version and fix any build errors that occur.
+1. Run `npm install --lockfile-version 2` to build with the new version and fix any build errors that occur.
 
 1. Run unit tests: `npm run test`
 
-1. Run end to end tests: `npm run test:e2e`. This requires running kafka & zookeeper locally.
-
 1. Update the version numbers referenced in the [`README.md`](https://github.com/Blizzard/node-rdkafka/blob/master/README.md) file to the new version.
 
 ## Publishing new npm version

README.md

@@ -17,7 +17,7 @@ I am looking for *your* help to make this project even better! If you're interes
 
 The `node-rdkafka` library is a high-performance NodeJS client for [Apache Kafka](http://kafka.apache.org/) that wraps the native  [librdkafka](https://github.com/edenhill/librdkafka) library.  All the complexity of balancing writes across partitions and managing (possibly ever-changing) brokers should be encapsulated in the library.
 
-__This library currently uses `librdkafka` version `1.9.2`.__
+__This library currently uses `librdkafka` version `2.3.0`.__
 
 ## Reference Docs
 
@@ -60,11 +60,7 @@ Using Alpine Linux? Check out the [docs](https://github.com/Blizzard/node-rdkafk
 
 ### Windows
 
-<<<<<<< HEAD
-Windows build **is not** compiled from `librdkafka` source but it is rather linked against the appropriate version of [NuGet librdkafka.redist](https://www.nuget.org/packages/librdkafka.redist/) static binary that gets downloaded from `https://globalcdn.nuget.org/packages/librdkafka.redist.1.8.2.nupkg` during installation. This download link can be changed using the environment variable `NODE_RDKAFKA_NUGET_BASE_URL` that defaults to `https://globalcdn.nuget.org/packages/` when it's no set.
-=======
-Windows build **is not** compiled from `librdkafka` source but it is rather linked against the appropriate version of [NuGet librdkafka.redist](https://www.nuget.org/packages/librdkafka.redist/) static binary that gets downloaded from `https://globalcdn.nuget.org/packages/librdkafka.redist.1.9.2.nupkg` during installation. This download link can be changed using the environment variable `NODE_RDKAFKA_NUGET_BASE_URL` that defaults to `https://globalcdn.nuget.org/packages/` when it's no set.
->>>>>>> 52b40e99abc811b2c4be1d3e62dd021e4bb1f6d4
+Windows build **is not** compiled from `librdkafka` source but it is rather linked against the appropriate version of [NuGet librdkafka.redist](https://www.nuget.org/packages/librdkafka.redist/) static binary that gets downloaded from `https://globalcdn.nuget.org/packages/librdkafka.redist.2.3.0.nupkg` during installation. This download link can be changed using the environment variable `NODE_RDKAFKA_NUGET_BASE_URL` that defaults to `https://globalcdn.nuget.org/packages/` when it's no set.
 
 Requirements:
  * [node-gyp for Windows](https://github.com/nodejs/node-gyp#on-windows)
@@ -96,12 +92,12 @@ npm install node-rdkafka
 To use the module, you must `require` it.
 
 ```js
-var Kafka = require('node-rdkafka');
+const Kafka = require('node-rdkafka');
 ```
 
 ## Configuration
 
-You can pass many configuration options to `librdkafka`.  A full list can be found in `librdkafka`'s [Configuration.md](https://github.com/edenhill/librdkafka/blob/v1.9.2/CONFIGURATION.md)
+You can pass many configuration options to `librdkafka`.  A full list can be found in `librdkafka`'s [Configuration.md](https://github.com/edenhill/librdkafka/blob/v2.3.0/CONFIGURATION.md)
 
 Configuration keys that have the suffix `_cb` are designated as callbacks. Some
 of these keys are informational and you can choose to opt-in (for example, `dr_cb`). Others are callbacks designed to
@@ -136,25 +132,25 @@ You can also get the version of `librdkafka`
 const Kafka = require('node-rdkafka');
 console.log(Kafka.librdkafkaVersion);
 
-// #=> 1.9.2
+// #=> 2.3.0
 ```
 
 ## Sending Messages
 
 A `Producer` sends messages to Kafka.  The `Producer` constructor takes a configuration object, as shown in the following example:
 
 ```js
-var producer = new Kafka.Producer({
+const producer = new Kafka.Producer({
   'metadata.broker.list': 'kafka-host1:9092,kafka-host2:9092'
 });
 ```
 
-A `Producer` requires only `metadata.broker.list` (the Kafka brokers) to be created.  The values in this list are separated by commas.  For other configuration options, see the [Configuration.md](https://github.com/edenhill/librdkafka/blob/v1.9.2/CONFIGURATION.md) file described previously.
+A `Producer` requires only `metadata.broker.list` (the Kafka brokers) to be created.  The values in this list are separated by commas.  For other configuration options, see the [Configuration.md](https://github.com/edenhill/librdkafka/blob/v2.3.0/CONFIGURATION.md) file described previously.
 
 The following example illustrates a list with several `librdkafka` options set.
 
 ```js
-var producer = new Kafka.Producer({
+const producer = new Kafka.Producer({
   'client.id': 'kafka',
   'metadata.broker.list': 'localhost:9092',
   'compression.codec': 'gzip',
@@ -175,14 +171,14 @@ You can easily use the `Producer` as a writable stream immediately after creatio
 ```js
 // Our producer with its Kafka brokers
 // This call returns a new writable stream to our topic 'topic-name'
-var stream = Kafka.Producer.createWriteStream({
+const stream = Kafka.Producer.createWriteStream({
   'metadata.broker.list': 'kafka-host1:9092,kafka-host2:9092'
 }, {}, {
   topic: 'topic-name'
 });
 
 // Writes a message to the stream
-var queuedSuccess = stream.write(Buffer.from('Awesome message'));
+const queuedSuccess = stream.write(Buffer.from('Awesome message'));
 
 if (queuedSuccess) {
   console.log('We queued our message!');
@@ -194,7 +190,7 @@ if (queuedSuccess) {
 
 // NOTE: MAKE SURE TO LISTEN TO THIS IF YOU WANT THE STREAM TO BE DURABLE
 // Otherwise, any error will bubble up as an uncaught exception.
-stream.on('error', function (err) {
+stream.on('error', (err) => {
   // Here's where we'll know if something went wrong sending to Kafka
   console.error('Error in our kafka stream');
   console.error(err);
@@ -209,7 +205,7 @@ The Standard API is more performant, particularly when handling high volumes of
 However, it requires more manual setup to use. The following example illustrates its use:
 
 ```js
-var producer = new Kafka.Producer({
+const producer = new Kafka.Producer({
   'metadata.broker.list': 'localhost:9092',
   'dr_cb': true
 });
@@ -218,7 +214,7 @@ var producer = new Kafka.Producer({
 producer.connect();
 
 // Wait for the ready event before proceeding
-producer.on('ready', function() {
+producer.on('ready', () => {
   try {
     producer.produce(
       // Topic to send the message to
@@ -243,7 +239,7 @@ producer.on('ready', function() {
 });
 
 // Any errors we encounter, including connection errors
-producer.on('event.error', function(err) {
+producer.on('event.error', (err) => {
   console.error('Error from producer');
   console.error(err);
 })
@@ -283,7 +279,7 @@ Some configuration properties that end in `_cb` indicate that an event should be
 The following example illustrates an event:
 
 ```js
-var producer = new Kafka.Producer({
+const producer = new Kafka.Producer({
   'client.id': 'my-client', // Specifies an identifier to use to help trace activity in Kafka
   'metadata.broker.list': 'localhost:9092', // Connect to a Kafka instance on localhost
   'dr_cb': true // Specifies that we want a delivery-report event to be generated
@@ -292,7 +288,7 @@ var producer = new Kafka.Producer({
 // Poll for events every 100 ms
 producer.setPollInterval(100);
 
-producer.on('delivery-report', function(err, report) {
+producer.on('delivery-report', (err, report) => {
   // Report of delivery statistics here:
   //
   console.log(report);
@@ -317,7 +313,7 @@ The following table describes types of events.
 The higher level producer is a variant of the producer which can propagate callbacks to you upon message delivery.
 
 ```js
-var producer = new Kafka.HighLevelProducer({
+const producer = new Kafka.HighLevelProducer({
   'metadata.broker.list': 'localhost:9092',
 });
 ```
@@ -334,7 +330,7 @@ producer.produce(topicName, null, Buffer.from('alliance4ever'), null, Date.now()
 Additionally you can add serializers to modify the value of a produce for a key or value before it is sent over to Kafka.
 
 ```js
-producer.setValueSerializer(function(value) {
+producer.setValueSerializer((value) => {
   return Buffer.from(JSON.stringify(value));
 });
 ```
@@ -346,7 +342,7 @@ Otherwise the behavior of the class should be exactly the same.
 To read messages from Kafka, you use a `KafkaConsumer`.  You instantiate a `KafkaConsumer` object as follows:
 
 ```js
-var consumer = new Kafka.KafkaConsumer({
+const consumer = new Kafka.KafkaConsumer({
   'group.id': 'kafka',
   'metadata.broker.list': 'localhost:9092',
 }, {});
@@ -361,10 +357,10 @@ The `group.id` and `metadata.broker.list` properties are required for a consumer
 Rebalancing is managed internally by `librdkafka` by default. If you would like to override this functionality, you may provide your own logic as a rebalance callback.
 
 ```js
-var consumer = new Kafka.KafkaConsumer({
+const consumer = new Kafka.KafkaConsumer({
   'group.id': 'kafka',
   'metadata.broker.list': 'localhost:9092',
-  'rebalance_cb': function(err, assignment) {
+  'rebalance_cb': (err, assignment) => {
 
     if (err.code === Kafka.CODES.ERRORS.ERR__ASSIGN_PARTITIONS) {
       // Note: this can throw when you are disconnected. Take care and wrap it in
@@ -389,10 +385,10 @@ var consumer = new Kafka.KafkaConsumer({
 When you commit in `node-rdkafka`, the standard way is to queue the commit request up with the next `librdkafka` request to the broker. When doing this, there isn't a way to know the result of the commit. Luckily there is another callback you can listen to to get this information
 
 ```js
-var consumer = new Kafka.KafkaConsumer({
+const consumer = new Kafka.KafkaConsumer({
   'group.id': 'kafka',
   'metadata.broker.list': 'localhost:9092',
-  'offset_commit_cb': function(err, topicPartitions) {
+  'offset_commit_cb': (err, topicPartitions) => {
 
     if (err) {
       // There was an error committing
@@ -430,11 +426,11 @@ The stream API is the easiest way to consume messages. The following example ill
 
 ```js
 // Read from the librdtesting-01 topic... note that this creates a new stream on each call!
-var stream = KafkaConsumer.createReadStream(globalConfig, topicConfig, {
+const stream = KafkaConsumer.createReadStream(globalConfig, topicConfig, {
   topics: ['librdtesting-01']
 });
 
-stream.on('data', function(message) {
+stream.on('data', (message) => {
   console.log('Got message');
   console.log(message.value.toString());
 });
@@ -459,15 +455,15 @@ The following example illustrates flowing mode:
 consumer.connect();
 
 consumer
-  .on('ready', function() {
+  .on('ready', () => {
     consumer.subscribe(['librdtesting-01']);
 
     // Consume from the librdtesting-01 topic. This is what determines
     // the mode we are running in. By not specifying a callback (or specifying
     // only a callback) we get messages as soon as they are available.
     consumer.consume();
   })
-  .on('data', function(data) {
+  .on('data', (data) => {
     // Output the actual message contents
     console.log(data.value.toString());
   });
@@ -478,17 +474,17 @@ The following example illustrates non-flowing mode:
 consumer.connect();
 
 consumer
-  .on('ready', function() {
+  .on('ready', () => {
     // Subscribe to the librdtesting-01 topic
     // This makes subsequent consumes read from that topic.
     consumer.subscribe(['librdtesting-01']);
 
     // Read one message every 1000 milliseconds
-    setInterval(function() {
+    setInterval(() => {
       consumer.consume(1);
     }, 1000);
   })
-  .on('data', function(data) {
+  .on('data', (data) => {
     console.log('Message found!  Contents below.');
     console.log(data.value.toString());
   });
@@ -528,15 +524,15 @@ The following table lists events for this API.
 Some times you find yourself in the situation where you need to know the latest (and earliest) offset for one of your topics. Connected producers and consumers both allow you to query for these through `queryWaterMarkOffsets` like follows:
 
 ```js
-var timeout = 5000, partition = 0;
-consumer.queryWatermarkOffsets('my-topic', partition, timeout, function(err, offsets) {
-  var high = offsets.highOffset;
-  var low = offsets.lowOffset;
+const timeout = 5000, partition = 0;
+consumer.queryWatermarkOffsets('my-topic', partition, timeout, (err, offsets) => {
+  const high = offsets.highOffset;
+  const low = offsets.lowOffset;
 });
 
-producer.queryWatermarkOffsets('my-topic', partition, timeout, function(err, offsets) {
-  var high = offsets.highOffset;
-  var low = offsets.lowOffset;
+producer.queryWatermarkOffsets('my-topic', partition, timeout, (err, offsets) => {
+  const high = offsets.highOffset;
+  const low = offsets.lowOffset;
 });
 
 An error will be returned if the client was not connected or the request timed out within the specified interval.
@@ -582,12 +578,12 @@ When fetching metadata for a specific topic, if a topic reference does not exist
 Please see the documentation on `Client.getMetadata` if you want to set configuration parameters, e.g. `acks`, on a topic to produce messages to.
 
 ```js
-var opts = {
+const opts = {
   topic: 'librdtesting-01',
   timeout: 10000
 };
 
-producer.getMetadata(opts, function(err, metadata) {
+producer.getMetadata(opts, (err, metadata) => {
   if (err) {
     console.error('Error getting metadata');
     console.error(err);
@@ -620,7 +616,7 @@ client.createTopic({
   topic: topicName,
   num_partitions: 1,
   replication_factor: 1
-}, function(err) {
+}, (err) => {
   // Done!
 });
 ```

ci/build-and-publish.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -ex
+
+# To prebuild images for arm platform run these on your local machine
+# this would cover everything we need (muslc / libc + 3 node versions)
+# Example: NODE_VER=18 platform=-rdkafka ./ci/build-and-publish.sh
+# Example: NODE_VER=20 platform=-rdkafka ./ci/build-and-publish.sh
+# Example: NODE_VER=21 platform=-rdkafka ./ci/build-and-publish.sh
+# Example: NODE_VER=18 platform=-debian-rdkafka ./ci/build-and-publish.sh
+# Example: NODE_VER=20 platform=-debian-rdkafka ./ci/build-and-publish.sh
+# Example: NODE_VER=21 platform=-debian-rdkafka ./ci/build-and-publish.sh
+
+cp ~/.env.aws-s3-credentials .env
+env IMAGE_TAG=${NODE_VER}${platform} UID=${UID} PNPM_STORE=$(pnpm config get store-dir) docker-compose up -d
+docker-compose exec tester pnpm i --frozen-lockfile --prefer-offline --ignore-scripts
+docker-compose exec tester pnpm binary:build
+docker-compose exec tester pnpm binary:package
+docker-compose exec tester pnpm binary:test
+docker-compose exec tester pnpm binary:publish