Data Structures | Macros | Functions
storagedir.c File Reference

An abstraction for a directory full of similar files. More...

#include "lib/fs/storagedir.h"
#include "lib/container/smartlist.h"
#include "lib/encoding/confline.h"
#include "lib/fs/dir.h"
#include "lib/fs/files.h"
#include "lib/fs/mmap.h"
#include "lib/log/escape.h"
#include "lib/log/log.h"
#include "lib/log/util_bug.h"
#include "lib/malloc/malloc.h"
#include "lib/memarea/memarea.h"
#include "lib/sandbox/sandbox.h"
#include "lib/string/printf.h"
#include "lib/string/util_string.h"
#include <stdlib.h>
#include <errno.h>
#include <string.h>

Go to the source code of this file.

Data Structures

struct  storage_dir_t
struct  shrinking_dir_entry_t


#define FNAME_MIN_NUM   1000


storage_dir_tstorage_dir_new (const char *dirname, int max_files)
void storage_dir_free_ (storage_dir_t *d)
int storage_dir_register_with_sandbox (storage_dir_t *d, sandbox_cfg_t **cfg)
static void storage_dir_clean_tmpfiles (storage_dir_t *d)
static int storage_dir_rescan (storage_dir_t *d)
const smartlist_tstorage_dir_list (storage_dir_t *d)
uint64_t storage_dir_get_usage (storage_dir_t *d)
tor_mmap_tstorage_dir_map (storage_dir_t *d, const char *fname)
uint8_t * storage_dir_read (storage_dir_t *d, const char *fname, int bin, size_t *sz_out)
static char * find_unused_fname (storage_dir_t *d)
static int storage_dir_save_chunks_to_file (storage_dir_t *d, const smartlist_t *chunks, int binary, char **fname_out)
int storage_dir_save_bytes_to_file (storage_dir_t *d, const uint8_t *data, size_t length, int binary, char **fname_out)
int storage_dir_save_string_to_file (storage_dir_t *d, const char *str, int binary, char **fname_out)
int storage_dir_save_labeled_to_file (storage_dir_t *d, const config_line_t *labels, const uint8_t *data, size_t length, char **fname_out)
tor_mmap_tstorage_dir_map_labeled (storage_dir_t *dir, const char *fname, config_line_t **labels_out, const uint8_t **data_out, size_t *sz_out)
uint8_t * storage_dir_read_labeled (storage_dir_t *dir, const char *fname, config_line_t **labels_out, size_t *sz_out)
static void storage_dir_reduce_usage (storage_dir_t *d, uint64_t removed_file_size)
void storage_dir_remove_file (storage_dir_t *d, const char *fname)
static int shrinking_dir_entry_compare (const void *a_, const void *b_)
int storage_dir_shrink (storage_dir_t *d, uint64_t target_size, int min_to_remove)
int storage_dir_remove_all (storage_dir_t *d)
int storage_dir_get_max_files (storage_dir_t *d)

Detailed Description

An abstraction for a directory full of similar files.

Storagedirs are used by our consensus cache code, and may someday also get used for unparseable objects. A large part of the need for this type is to work around the limitations in our sandbox code, where all filenames need to be registered in advance.

Definition in file storagedir.c.

Macro Definition Documentation


#define FNAME_MIN_NUM   1000

Definition at line 44 of file storagedir.c.

Function Documentation

◆ find_unused_fname()

static char * find_unused_fname ( storage_dir_t d)

Helper: Find an unused filename within the directory

Definition at line 252 of file storagedir.c.

Referenced by storage_dir_save_chunks_to_file().

◆ shrinking_dir_entry_compare()

static int shrinking_dir_entry_compare ( const void *  a_,
const void *  b_ 

Helper: use with qsort to sort shrinking_dir_entry_t structs.

Definition at line 518 of file storagedir.c.

◆ storage_dir_clean_tmpfiles()

static void storage_dir_clean_tmpfiles ( storage_dir_t d)

Remove all files in d whose names end with ".tmp".

Requires that the contents field of d is set.

Definition at line 133 of file storagedir.c.

◆ storage_dir_free_()

void storage_dir_free_ ( storage_dir_t d)

Drop all in-RAM storage for d. Does not delete any files.

Definition at line 83 of file storagedir.c.

◆ storage_dir_get_max_files()

int storage_dir_get_max_files ( storage_dir_t d)

Return the largest number of non-temporary files we're willing to store in d.

Definition at line 603 of file storagedir.c.

◆ storage_dir_get_usage()

uint64_t storage_dir_get_usage ( storage_dir_t d)

Return the total number of bytes used for storage in d.

Definition at line 190 of file storagedir.c.

Referenced by storage_dir_shrink().

◆ storage_dir_list()

const smartlist_t * storage_dir_list ( storage_dir_t d)

Return a smartlist containing the filenames within d.

Definition at line 179 of file storagedir.c.

Referenced by consensus_cache_rescan(), and storage_dir_get_usage().

◆ storage_dir_map()

tor_mmap_t * storage_dir_map ( storage_dir_t d,
const char *  fname 

Mmap a specified file within d.

On failure, return NULL and set errno as for tor_mmap_file().

Definition at line 215 of file storagedir.c.

Referenced by storage_dir_map_labeled().

◆ storage_dir_map_labeled()

tor_mmap_t * storage_dir_map_labeled ( storage_dir_t dir,
const char *  fname,
config_line_t **  labels_out,
const uint8_t **  data_out,
size_t *  sz_out 

Map a file that was created with storage_dir_save_labeled_to_file(). On failure, return NULL. On success, write a set of newly allocated labels into *labels_out, a pointer to the data into *data_out, and the data's size into *sz_out. On success, also return a tor_mmap_t object whose contents should not be used – it needs to be kept around, though, for as long as data_out is going to be valid.

On failure, set errno as for tor_mmap_file() if the file was missing or empty, and set errno to EINVAL if the file was not in the labeled format expected.

Definition at line 399 of file storagedir.c.

Referenced by consensus_cache_entry_map(), consensus_cache_rescan(), and storage_dir_read_labeled().

◆ storage_dir_new()

storage_dir_t * storage_dir_new ( const char *  dirname,
int  max_files 

Create or open a new storage directory at dirname, with capacity for up to max_files files.

Definition at line 68 of file storagedir.c.

◆ storage_dir_read()

uint8_t * storage_dir_read ( storage_dir_t d,
const char *  fname,
int  bin,
size_t *  sz_out 

Read a file within d into a newly allocated buffer. Set *sz_out to its size.

Definition at line 230 of file storagedir.c.

◆ storage_dir_read_labeled()

uint8_t * storage_dir_read_labeled ( storage_dir_t dir,
const char *  fname,
config_line_t **  labels_out,
size_t *  sz_out 

As storage_dir_map_labeled, but return a new byte array containing the data.

Definition at line 435 of file storagedir.c.

◆ storage_dir_reduce_usage()

static void storage_dir_reduce_usage ( storage_dir_t d,
uint64_t  removed_file_size 

Definition at line 453 of file storagedir.c.

◆ storage_dir_register_with_sandbox()

int storage_dir_register_with_sandbox ( storage_dir_t d,
sandbox_cfg_t **  cfg 

Tell the sandbox (if any) configured by cfg to allow the operations that d will need.

The presence of this function is why we need an upper limit on the number of files in a storage_dir_t: we need to approve file operations one by one.

Definition at line 104 of file storagedir.c.

Referenced by consensus_cache_register_with_sandbox().

◆ storage_dir_remove_all()

int storage_dir_remove_all ( storage_dir_t d)

Remove all files in d.

Definition at line 593 of file storagedir.c.

◆ storage_dir_remove_file()

void storage_dir_remove_file ( storage_dir_t d,
const char *  fname 

Remove the file called fname from d.

Definition at line 480 of file storagedir.c.

◆ storage_dir_rescan()

static int storage_dir_rescan ( storage_dir_t d)

Re-scan the directory d to learn its contents.

Definition at line 160 of file storagedir.c.

Referenced by find_unused_fname(), storage_dir_list(), and storage_dir_shrink().

◆ storage_dir_save_bytes_to_file()

int storage_dir_save_bytes_to_file ( storage_dir_t d,
const uint8_t *  data,
size_t  length,
int  binary,
char **  fname_out 

Try to write the length bytes at data into a new file in d. On success, return 0 and set *fname_out to a newly allocated string containing the filename. On failure, return -1.

Definition at line 311 of file storagedir.c.

Referenced by storage_dir_save_string_to_file().

◆ storage_dir_save_chunks_to_file()

static int storage_dir_save_chunks_to_file ( storage_dir_t d,
const smartlist_t chunks,
int  binary,
char **  fname_out 

Helper: As storage_dir_save_bytes_to_file, but store a smartlist of sized_chunk_t rather than a single byte array.

Definition at line 275 of file storagedir.c.

Referenced by storage_dir_save_bytes_to_file(), and storage_dir_save_labeled_to_file().

◆ storage_dir_save_labeled_to_file()

int storage_dir_save_labeled_to_file ( storage_dir_t d,
const config_line_t labels,
const uint8_t *  data,
size_t  length,
char **  fname_out 

As storage_dir_save_bytes_to_file, but associates the data with the key-value pairs in labels. Files stored in this format can be recovered with storage_dir_map_labeled() or storage_dir_read_labeled().

Definition at line 345 of file storagedir.c.

Referenced by consensus_cache_add().

◆ storage_dir_save_string_to_file()

int storage_dir_save_string_to_file ( storage_dir_t d,
const char *  str,
int  binary,
char **  fname_out 

As storage_dir_save_bytes_to_file, but saves a NUL-terminated string str.

Definition at line 330 of file storagedir.c.

◆ storage_dir_shrink()

int storage_dir_shrink ( storage_dir_t d,
uint64_t  target_size,
int  min_to_remove 

Try to free space by removing the oldest files in d. Delete until no more than target_size bytes are left, and at least min_to_remove files have been removed... or until there is nothing left to remove.

Return 0 on success; -1 on failure.

Definition at line 540 of file storagedir.c.

Referenced by storage_dir_remove_all().