src/n_disk_cache.c

A bounded cache of files in one directory, keyed by content. More...

Structures

	Name
struct	_N_DC_GROUP_ENTRY one entry of a group prune: the entry and the cache it is in
struct	_N_DC_ENTRY one entry found by a walk of the directory
struct	N_DISK_CACHE a cache: only where it is

Functions

	Name
int	_n_dc_walk(const N_DISK_CACHE * cache, _N_DC_ENTRY ** out) helper: every entry of the cache, in a new array the caller frees.
int	_n_dc_replace(const char * from, const char * to) helper: rename over an existing target
int	_n_dc_remove(const char * path) helper: remove a UTF-8 path.
int	_n_dc_owned(const char * name) helper: is name one this module could have made: 32 lowercase hex digits, then the end or a '.
int	_n_dc_older(const void * a, const void * b) qsort order: least recently used first
int	_n_dc_join(const N_DISK_CACHE * cache, const char * name, char * out, size_t out_len) helper: join the directory and a file name
int	_n_dc_is_dir(const char * path) helper: is path a directory
int	_n_dc_group_older(const void * a, const void * b) qsort order for a group: least recently used first
FILE *	_n_dc_fopen(const char * path, const char * mode) helper: fopen a UTF-8 path

Defines

	Name
	N_DISK_CACHE_SEED seed of the key hash.
	N_DISK_CACHE_PATH_MAX longest path this module builds

Detailed Description

A bounded cache of files in one directory, keyed by content.

Author: Castagnier Mickael

Version: 1.0

Date: 24/09/2026

Functions Documentation

function _n_dc_walk

static int _n_dc_walk(
    const N_DISK_CACHE * cache,
    _N_DC_ENTRY ** out
)

helper: every entry of the cache, in a new array the caller frees.

Returns the count, or -1 when the directory cannot be read.

function _n_dc_replace

static int _n_dc_replace(
    const char * from,
    const char * to
)

helper: rename over an existing target

function _n_dc_remove

static int _n_dc_remove(
    const char * path
)

helper: remove a UTF-8 path.

TRUE when it is gone or was never there.

function _n_dc_owned

static int _n_dc_owned(
    const char * name
)

helper: is name one this module could have made: 32 lowercase hex digits, then the end or a '.

'

function _n_dc_older

static int _n_dc_older(
    const void * a,
    const void * b
)

qsort order: least recently used first

function _n_dc_join

static int _n_dc_join(
    const N_DISK_CACHE * cache,
    const char * name,
    char * out,
    size_t out_len
)

helper: join the directory and a file name

function _n_dc_is_dir

static int _n_dc_is_dir(
    const char * path
)

helper: is path a directory

function _n_dc_group_older

static int _n_dc_group_older(
    const void * a,
    const void * b
)

qsort order for a group: least recently used first

function _n_dc_fopen

static FILE * _n_dc_fopen(
    const char * path,
    const char * mode
)

helper: fopen a UTF-8 path

Macros Documentation

define N_DISK_CACHE_SEED

#define N_DISK_CACHE_SEED 0x4E444331u

seed of the key hash.

Fixed, so a cache written by one build is read by the next.

define N_DISK_CACHE_PATH_MAX

#define N_DISK_CACHE_PATH_MAX 4096

longest path this module builds

Source code

/*
 * Nilorea Library
 * Copyright (C) 2005-2026 Castagnier Mickael
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 *
 * SPDX-License-Identifier: Apache-2.0
 */


#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>

#include "nilorea/n_disk_cache.h"
#include "nilorea/n_hash.h"
#include "nilorea/n_log.h"

#ifdef __windows__
#include <windows.h>
#else
#include <dirent.h>
#include <sys/stat.h>
#include <unistd.h>
#endif

#define N_DISK_CACHE_SEED 0x4E444331u

#define N_DISK_CACHE_PATH_MAX 4096

struct N_DISK_CACHE {
    char* dir;
};

typedef struct {
    char name[N_DISK_CACHE_NAME_LEN + 64];
    uint64_t size;
    int64_t used;
} _N_DC_ENTRY;

#ifdef __windows__
static wchar_t* _n_dc_wide(const char* path) {
    int n = MultiByteToWideChar(CP_UTF8, 0, path, -1, NULL, 0);
    wchar_t* w = NULL;

    if (n <= 0) return NULL;
    w = (wchar_t*)malloc(sizeof(wchar_t) * (size_t)n);
    if (!w) return NULL;
    if (MultiByteToWideChar(CP_UTF8, 0, path, -1, w, n) <= 0) {
        free(w);
        return NULL;
    }
    return w;
}

static int64_t _n_dc_filetime(const FILETIME* ft) {
    uint64_t t = ((uint64_t)ft->dwHighDateTime << 32) | (uint64_t)ft->dwLowDateTime;

    if (t == 0) return 0;
    /* 100 ns ticks since 1601 */
    return (int64_t)(t / 10000000ULL) - 11644473600LL;
}
#endif

static FILE* _n_dc_fopen(const char* path, const char* mode) {
#ifdef __windows__
    wchar_t* wp = _n_dc_wide(path);
    wchar_t* wm = _n_dc_wide(mode);
    FILE* f = (wp && wm) ? _wfopen(wp, wm) : NULL;

    free(wp);
    free(wm);
    return f;
#else
    return fopen(path, mode);
#endif
}

static int _n_dc_remove(const char* path) {
#ifdef __windows__
    wchar_t* wp = _n_dc_wide(path);

    if (!wp) return FALSE;
    int ok = DeleteFileW(wp) || GetLastError() == ERROR_FILE_NOT_FOUND;
    free(wp);
    return ok ? TRUE : FALSE;
#else
    return (unlink(path) == 0 || errno == ENOENT) ? TRUE : FALSE;
#endif
}

static int _n_dc_replace(const char* from, const char* to) {
#ifdef __windows__
    wchar_t* wf = _n_dc_wide(from);
    wchar_t* wt = _n_dc_wide(to);
    int ok = (wf && wt && MoveFileExW(wf, wt, MOVEFILE_REPLACE_EXISTING)) ? TRUE : FALSE;

    free(wf);
    free(wt);
    return ok;
#else
    return rename(from, to) == 0 ? TRUE : FALSE;
#endif
}

static int _n_dc_is_dir(const char* path) {
#ifdef __windows__
    wchar_t* wp = _n_dc_wide(path);
    DWORD attr = wp ? GetFileAttributesW(wp) : INVALID_FILE_ATTRIBUTES;

    free(wp);
    return (attr != INVALID_FILE_ATTRIBUTES && (attr & FILE_ATTRIBUTE_DIRECTORY)) ? TRUE : FALSE;
#else
    struct stat st;

    return (stat(path, &st) == 0 && S_ISDIR(st.st_mode)) ? TRUE : FALSE;
#endif
}

static int _n_dc_owned(const char* name) {
    int i = 0;

    for (i = 0; i < N_DISK_CACHE_NAME_LEN; i++) {
        char c = name[i];

        if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'))) return FALSE;
    }
    return (name[N_DISK_CACHE_NAME_LEN] == '\0' || name[N_DISK_CACHE_NAME_LEN] == '.') ? TRUE : FALSE;
}

static int _n_dc_join(const N_DISK_CACHE* cache, const char* name, char* out, size_t out_len) {
    int n = snprintf(out, out_len, "%s/%s", cache->dir, name);

    return (n > 0 && (size_t)n < out_len) ? TRUE : FALSE;
}

static int _n_dc_walk(const N_DISK_CACHE* cache, _N_DC_ENTRY** out) {
    _N_DC_ENTRY* list = NULL;
    int count = 0, room = 0;

    *out = NULL;
#ifdef __windows__
    {
        char pattern[N_DISK_CACHE_PATH_MAX];
        wchar_t* wp = NULL;
        WIN32_FIND_DATAW fd;
        HANDLE h = INVALID_HANDLE_VALUE;

        if (!_n_dc_join(cache, "*", pattern, sizeof(pattern))) return -1;
        wp = _n_dc_wide(pattern);
        if (!wp) return -1;
        h = FindFirstFileW(wp, &fd);
        free(wp);
        if (h == INVALID_HANDLE_VALUE) return GetLastError() == ERROR_FILE_NOT_FOUND ? 0 : -1;
        do {
            char name[N_DISK_CACHE_NAME_LEN + 64];
            int64_t used = 0;

            if (fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) continue;
            /* an entry's name is ASCII; anything that does not convert, or
             * does not fit, is not one */
            if (WideCharToMultiByte(CP_UTF8, 0, fd.cFileName, -1, name, (int)sizeof(name), NULL, NULL) <= 0) continue;
            if (!_n_dc_owned(name)) continue;
            if (count == room) {
                _N_DC_ENTRY* grown = NULL;

                room = room ? room * 2 : 256;
                grown = (_N_DC_ENTRY*)realloc(list, sizeof(*list) * (size_t)room);
                if (!grown) {
                    free(list);
                    FindClose(h);
                    return -1;
                }
                list = grown;
            }
            snprintf(list[count].name, sizeof(list[count].name), "%s", name);
            list[count].size = ((uint64_t)fd.nFileSizeHigh << 32) | (uint64_t)fd.nFileSizeLow;
            used = _n_dc_filetime(&fd.ftLastAccessTime);
            list[count].used = used ? used : _n_dc_filetime(&fd.ftLastWriteTime);
            count++;
        } while (FindNextFileW(h, &fd));
        FindClose(h);
    }
#else
    {
        DIR* dp = opendir(cache->dir);
        const struct dirent* de = NULL;

        if (!dp) return -1;
        while ((de = readdir(dp)) != NULL) {
            char path[N_DISK_CACHE_PATH_MAX];
            struct stat st;
            size_t name_len = 0;

            if (!_n_dc_owned(de->d_name)) continue;
            name_len = strlen(de->d_name);
            if (name_len >= sizeof(list[0].name)) continue;
            if (!_n_dc_join(cache, de->d_name, path, sizeof(path))) continue;
            /* gone since readdir, or not a file: not an entry */
            if (stat(path, &st) != 0 || !S_ISREG(st.st_mode)) continue;
            if (count == room) {
                _N_DC_ENTRY* grown = NULL;

                room = room ? room * 2 : 256;
                grown = (_N_DC_ENTRY*)realloc(list, sizeof(*list) * (size_t)room);
                if (!grown) {
                    free(list);
                    closedir(dp);
                    return -1;
                }
                list = grown;
            }
            memcpy(list[count].name, de->d_name, name_len + 1);
            list[count].size = (uint64_t)st.st_size;
            list[count].used = (int64_t)(st.st_atime ? st.st_atime : st.st_mtime);
            count++;
        }
        closedir(dp);
    }
#endif
    *out = list;
    return count;
}

static int _n_dc_older(const void* a, const void* b) {
    int64_t ua = ((const _N_DC_ENTRY*)a)->used;
    int64_t ub = ((const _N_DC_ENTRY*)b)->used;

    return (ua > ub) - (ua < ub);
}

N_DISK_CACHE* n_disk_cache_open(const char* dir) {
    N_DISK_CACHE* cache = NULL;
    size_t len = 0;

    __n_assert(dir, return NULL);
    len = strlen(dir);
    /* the root alone would lose its only separator below */
    while (len > 1 && (dir[len - 1] == '/' || dir[len - 1] == '\\')) len--;
    if (len == 0 || len >= N_DISK_CACHE_PATH_MAX - N_DISK_CACHE_NAME_LEN - 64) {
        n_log(LOG_ERR, "disk cache: unusable directory name");
        return NULL;
    }
    Malloc(cache, N_DISK_CACHE, 1);
    __n_assert(cache, return NULL);
    Malloc(cache->dir, char, len + 1);
    if (!cache->dir) {
        Free(cache);
        return NULL;
    }
    memcpy(cache->dir, dir, len);
    cache->dir[len] = '\0';
    if (!_n_dc_is_dir(cache->dir)) {
        n_log(LOG_ERR, "disk cache: %s is not a directory", cache->dir);
        Free(cache->dir);
        Free(cache);
        return NULL;
    }
    return cache;
}

void n_disk_cache_close(N_DISK_CACHE** cache) {
    if (!cache || !*cache) return;
    Free((*cache)->dir);
    Free(*cache);
}

const char* n_disk_cache_dir(const N_DISK_CACHE* cache) {
    return cache ? cache->dir : "";
}

int n_disk_cache_path(const N_DISK_CACHE* cache, const void* key, size_t key_len, const char* ext, char* out, size_t out_len) {
    uint64_t hash[2] = {0, 0};
    char name[N_DISK_CACHE_NAME_LEN + 64];
    int n = 0;

    __n_assert(cache, return FALSE);
    __n_assert(key, return FALSE);
    __n_assert(out, return FALSE);
    if (!ext) ext = "";
    if (strchr(ext, '/') || strchr(ext, '\\') || strlen(ext) >= sizeof(name) - N_DISK_CACHE_NAME_LEN) return FALSE;
    MurmurHash3_x64_128(key, key_len, N_DISK_CACHE_SEED, hash);
    n = snprintf(name, sizeof(name), "%016llx%016llx%s", (unsigned long long)hash[0], (unsigned long long)hash[1], ext);
    if (n <= 0 || (size_t)n >= sizeof(name)) return FALSE;
    return _n_dc_join(cache, name, out, out_len);
}

int n_disk_cache_put(const N_DISK_CACHE* cache, const void* key, size_t key_len, const char* ext, const void* data, size_t len) {
    char path[N_DISK_CACHE_PATH_MAX];
    char tmp[N_DISK_CACHE_PATH_MAX + 8];
    FILE* f = NULL;
    int ok = TRUE;

    __n_assert(cache, return FALSE);
    if (!data && len) return FALSE;
    if (!n_disk_cache_path(cache, key, key_len, ext, path, sizeof(path))) return FALSE;
    /* A sibling then a rename, so a reader sees the old entry or the new one.
     * Not flushed to stable storage: a cache entry lost to a power cut is a
     * miss, which is what a cache is for. The temporary's name is still an
     * entry's, so a crash's leftover is pruned like any other. */
    snprintf(tmp, sizeof(tmp), "%s.tmp", path);
    f = _n_dc_fopen(tmp, "wb");
    if (!f) {
        n_log(LOG_DEBUG, "disk cache: cannot write %s", tmp);
        return FALSE;
    }
    if (len && fwrite(data, 1, len, f) != len) ok = FALSE;
    if (fclose(f) != 0) ok = FALSE;
    if (ok) ok = _n_dc_replace(tmp, path);
    if (!ok) {
        _n_dc_remove(tmp);
        n_log(LOG_DEBUG, "disk cache: cannot store %s", path);
    }
    return ok;
}

int n_disk_cache_get(const N_DISK_CACHE* cache, const void* key, size_t key_len, const char* ext, size_t max_len, unsigned char** data, size_t* len) {
    char path[N_DISK_CACHE_PATH_MAX];
    unsigned char* buf = NULL;
    FILE* f = NULL;
    long size = 0;

    __n_assert(data, return FALSE);
    __n_assert(len, return FALSE);
    *data = NULL;
    *len = 0;
    __n_assert(cache, return FALSE);
    if (!n_disk_cache_path(cache, key, key_len, ext, path, sizeof(path))) return FALSE;
    f = _n_dc_fopen(path, "rb");
    if (!f) return FALSE;
    if (fseek(f, 0, SEEK_END) != 0 || (size = ftell(f)) < 0 || (uint64_t)size > (uint64_t)max_len || fseek(f, 0, SEEK_SET) != 0) {
        fclose(f);
        return FALSE;
    }
    buf = (unsigned char*)malloc((size_t)size + 1);
    if (!buf) {
        fclose(f);
        return FALSE;
    }
    if (size && fread(buf, 1, (size_t)size, f) != (size_t)size) {
        free(buf);
        fclose(f);
        return FALSE;
    }
    fclose(f);
    buf[size] = '\0';
    *data = buf;
    *len = (size_t)size;
    return TRUE;
}

int n_disk_cache_remove(const N_DISK_CACHE* cache, const void* key, size_t key_len, const char* ext) {
    char path[N_DISK_CACHE_PATH_MAX];

    __n_assert(cache, return FALSE);
    if (!n_disk_cache_path(cache, key, key_len, ext, path, sizeof(path))) return FALSE;
    return _n_dc_remove(path);
}

int n_disk_cache_size(const N_DISK_CACHE* cache, uint64_t* total) {
    _N_DC_ENTRY* list = NULL;
    int count = 0, i = 0;

    __n_assert(cache, return -1);
    __n_assert(total, return -1);
    *total = 0;
    count = _n_dc_walk(cache, &list);
    for (i = 0; i < count; i++) *total += list[i].size;
    free(list);
    return count;
}

int n_disk_cache_prune(const N_DISK_CACHE* cache, uint64_t max_bytes) {
    _N_DC_ENTRY* list = NULL;
    uint64_t total = 0;
    int count = 0, i = 0, removed = 0;

    __n_assert(cache, return -1);
    count = _n_dc_walk(cache, &list);
    if (count < 0) return -1;
    for (i = 0; i < count; i++) total += list[i].size;
    if (total > max_bytes || max_bytes == 0) {
        qsort(list, (size_t)count, sizeof(*list), _n_dc_older);
        for (i = 0; i < count && (total > max_bytes || max_bytes == 0); i++) {
            char path[N_DISK_CACHE_PATH_MAX];

            if (!_n_dc_join(cache, list[i].name, path, sizeof(path))) continue;
            if (_n_dc_remove(path)) {
                total -= list[i].size;
                removed++;
            }
        }
    }
    free(list);
    if (removed) n_log(LOG_DEBUG, "disk cache: %s, removed %d entries, %llu bytes left", cache->dir, removed, (unsigned long long)total);
    return removed;
}

typedef struct {
    _N_DC_ENTRY entry;
    int owner;
} _N_DC_GROUP_ENTRY;

static int _n_dc_group_older(const void* a, const void* b) {
    return _n_dc_older(&((const _N_DC_GROUP_ENTRY*)a)->entry, &((const _N_DC_GROUP_ENTRY*)b)->entry);
}

int n_disk_cache_prune_group(const N_DISK_CACHE* const* caches, int count, uint64_t max_bytes) {
    _N_DC_GROUP_ENTRY* all = NULL;
    size_t total_count = 0, room = 0;
    uint64_t total = 0;
    int c = 0, removed = 0;

    __n_assert(caches, return -1);
    if (count <= 0) return -1;
    /* every directory is read before anything is removed, so a directory
     * that cannot be read leaves the others as they were */
    for (c = 0; c < count; c++) {
        _N_DC_ENTRY* list = NULL;
        int n = 0, k = 0;

        if (!caches[c]) continue;
        n = _n_dc_walk(caches[c], &list);
        if (n < 0) {
            free(all);
            return -1;
        }
        if (total_count + (size_t)n > room) {
            _N_DC_GROUP_ENTRY* grown = NULL;

            room = total_count + (size_t)n + 256;
            grown = (_N_DC_GROUP_ENTRY*)realloc(all, sizeof(*all) * room);
            if (!grown) {
                free(list);
                free(all);
                return -1;
            }
            all = grown;
        }
        for (k = 0; k < n; k++) {
            all[total_count].entry = list[k];
            all[total_count].owner = c;
            total += list[k].size;
            total_count++;
        }
        free(list);
    }
    /* nothing found is nothing to sort: all is still NULL then */
    if (all && total_count && (total > max_bytes || max_bytes == 0)) {
        size_t i = 0;

        qsort(all, total_count, sizeof(*all), _n_dc_group_older);
        for (i = 0; i < total_count && (total > max_bytes || max_bytes == 0); i++) {
            char path[N_DISK_CACHE_PATH_MAX];

            if (!_n_dc_join(caches[all[i].owner], all[i].entry.name, path, sizeof(path))) continue;
            if (_n_dc_remove(path)) {
                total -= all[i].entry.size;
                removed++;
            }
        }
    }
    free(all);
    if (removed) n_log(LOG_DEBUG, "disk cache: %d caches, removed %d entries, %llu bytes left", count, removed, (unsigned long long)total);
    return removed;
}

int n_disk_cache_expire(const N_DISK_CACHE* cache, int64_t max_age_seconds) {
    _N_DC_ENTRY* list = NULL;
    int64_t cutoff = 0;
    int count = 0, i = 0, removed = 0;

    __n_assert(cache, return -1);
    if (max_age_seconds <= 0) return 0;
    count = _n_dc_walk(cache, &list);
    if (count < 0) return -1;
    cutoff = (int64_t)time(NULL) - max_age_seconds;
    for (i = 0; i < count; i++) {
        char path[N_DISK_CACHE_PATH_MAX];

        if (list[i].used > cutoff) continue;
        if (_n_dc_join(cache, list[i].name, path, sizeof(path)) && _n_dc_remove(path)) removed++;
    }
    free(list);
    if (removed) n_log(LOG_DEBUG, "disk cache: %s, removed %d entries unused for %lld s", cache->dir, removed, (long long)max_age_seconds);
    return removed;
}