include/nilorea/n_kafka.h

Kafka generic produce and consume event header. More...

Structures

	Name
struct	N_KAFKA_EVENT structure of a KAFKA message
struct	N_KAFKA structure of a KAFKA consumer or producer handle

Detailed Description

Kafka generic produce and consume event header.

Author: Castagnier Mickael

Version: 1.0

Date: 23/11/2022

Source code

/*
 * Nilorea Library
 * Copyright (C) 2005-2026 Castagnier Mickael
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 *
 * SPDX-License-Identifier: Apache-2.0
 */


#ifndef __N_KAFKA
#define __N_KAFKA

#ifdef __cplusplus
extern "C" {
#endif


#include "nilorea/n_log.h"
#include "nilorea/n_network.h"
#include "cJSON.h"

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <locale.h>
#include <libgen.h>
#include <errno.h>
#include <time.h>
#include <unistd.h>

#include "rdkafka.h"

#define N_KAFKA_EVENT_QUEUED 0
#define N_KAFKA_EVENT_WAITING_ACK 1
#define N_KAFKA_EVENT_ERROR 2
#define N_KAFKA_EVENT_OK 4
#define N_KAFKA_EVENT_CREATED 5

typedef struct N_KAFKA_EVENT {
    N_STR* event_string;
    N_STR* event_files_to_delete;
    char* from_topic;
    unsigned int status;
    rd_kafka_headers_t* rd_kafka_headers;
    LIST* received_headers;
    int schema_id;
    struct N_KAFKA* parent_table;
    time_t error_time;
} N_KAFKA_EVENT;

typedef struct N_KAFKA {
    LIST* events_to_send;
    LIST* received_events;
    char* groupid;
    char** topics;
    char* event_cmd;
    rd_kafka_topic_partition_list_t* subscription;
    rd_kafka_conf_t* rd_kafka_conf;
    rd_kafka_t* rd_kafka_handle;
    char* topic;
    char* bootstrap_servers;
    rd_kafka_topic_t* rd_kafka_topic;
    N_STR* errstr;
    int schema_id;
    pthread_rwlock_t rwlock;
    int mode;
    cJSON* configuration;
    int32_t poll_timeout;
    int32_t poll_interval;
    int32_t monitored_directory_interval;
    int polling_thread_status;
    pthread_t polling_thread;
    size_t nb_queued;
    size_t nb_waiting;
    size_t nb_error;
    int32_t event_consumption_enabled;
    int32_t event_production_enabled;
    int32_t error_timeout;
    int is_transactional;
    char* tosend_dir;
    char* sended_dir;
} N_KAFKA;

int32_t n_kafka_get_schema_from_char(const char* string);
int32_t n_kafka_get_schema_from_nstr(const N_STR* string);
int n_kafka_put_schema_in_char(char* string, int schema_id);
int n_kafka_put_schema_in_nstr(N_STR* string, int schema_id);

int n_kafka_get_status(N_KAFKA* kafka, size_t* nb_queued, size_t* nb_waiting, size_t* nb_error);
void n_kafka_delete(N_KAFKA* kafka);
N_KAFKA* n_kafka_new(int32_t poll_timeout, int32_t poll_interval, size_t errstr_len);
int n_kafka_set_error_timeout(N_KAFKA* kafka, int32_t error_timeout);
int n_kafka_set_sended_dir(N_KAFKA* kafka, const char* tosend_dir, const char* sended_dir);
N_KAFKA* n_kafka_load_config(char* config_file, int mode);

int n_kafka_new_headers(N_KAFKA_EVENT* event, size_t count);
int n_kafka_add_header_ex(N_KAFKA_EVENT* event, char* key, size_t key_length, char* value, size_t value_length);
int n_kafka_add_header(N_KAFKA_EVENT* event, N_STR* key, N_STR* value);

int n_kafka_produce(N_KAFKA* kafka, N_KAFKA_EVENT* event);

N_KAFKA_EVENT* n_kafka_new_event(int schema_id);
N_KAFKA_EVENT* n_kafka_new_event_from_char(const char* string, size_t written, int schema_id);
N_KAFKA_EVENT* n_kafka_new_event_from_string(const N_STR* string, int schema_id);
N_KAFKA_EVENT* n_kafka_new_event_from_file(char* filename, int schema_id);
void n_kafka_event_destroy_ptr(void* event);
int n_kafka_event_destroy(N_KAFKA_EVENT** event);

int n_kafka_poll(N_KAFKA* kafka);
int n_kafka_start_polling_thread(N_KAFKA* kafka);
int n_kafka_stop_polling_thread(N_KAFKA* kafka);
int n_kafka_enable_event_consumption(N_KAFKA* kafka);
int n_kafka_disable_event_consumption(N_KAFKA* kafka);
int n_kafka_enable_event_production(N_KAFKA* kafka);
int n_kafka_disable_event_production(N_KAFKA* kafka);
int n_kafka_dump_unprocessed(N_KAFKA* kafka, char* directory);
int n_kafka_load_unprocessed(N_KAFKA* kafka, const char* directory);

N_KAFKA_EVENT* n_kafka_get_event(N_KAFKA* kafka);


#ifdef __cplusplus
}
#endif

#endif  // header guard