huge.c File Reference

Detailed Description

HUGE support (Hash/URN Gnutella Extension).

Author:: Raphael Manfredi

Date:: 2002-2003

Author:: Ch. Tronche (http://tronche.com/)

Date:: 2002-04-28

#include "common.h"
#include "huge.h"
#include "share.h"
#include "gmsg.h"
#include "dmesh.h"
#include "version.h"
#include "settings.h"
#include "lib/atoms.h"
#include "lib/base32.h"
#include "lib/bg.h"
#include "lib/file.h"
#include "lib/header.h"
#include "lib/sha1.h"
#include "lib/tm.h"
#include "lib/urn.h"
#include "lib/walloc.h"
#include "if/gnet_property.h"
#include "if/gnet_property_priv.h"
#include "lib/override.h"

Data Structures

struct sha1_cache_entry

There's an in-core cache (the GHashTable sha1_cache), and a persistent copy (normally in ~/.gtk-gnutella/sha1_cache). More...

struct file_sha1

This is a file waiting either for the digest to be computer, or when computed to be retrofit into the share record. More...

struct sha1_computation_context

Defines

#define HASH_BLOCK_SHIFT   12 /**< Power of two of hash unit credit */

Power of two of hash unit credit.

#define HASH_BUF_SIZE   65536 /**< Size of a the reading buffer */

Size of a the reading buffer.

#define SHA1_MAGIC   0xa1a1a1a1U

Functions

RCSID ("$Id:huge.c, v 1.24 2005/09/10 08:17:28 daichik Exp $")

void copy_sha1 (char *dest, const char *source)

copy_sha1

void update_volatile_cache (struct sha1_cache_entry *sha1_cached_entry, filesize_t size, time_t mtime, const char *digest)

update_volatile_cache

void add_volatile_cache_entry (const char *filename, filesize_t size, time_t mtime, const char *digest, gboolean known_to_be_shared)

add_volatile_cache_entry

void cache_entry_print (FILE *f, const char *filename, const gchar *digest, filesize_t size, time_t mtime)

void add_persistent_cache_entry (const char *filename, filesize_t size, time_t mtime, const char *digest)

Add an entry to the persistent cache.

void dump_cache_one_entry (gpointer unused_key, gpointer value, gpointer udata)

Dump one (in-memory) cache into the persistent cache.

void dump_cache (void)

Dump the whole in-memory cache onto disk.

void parse_and_append_cache_entry (char *line)

This function is used to read the disk cache into memory.

void sha1_read_cache (void)

Read the whole persistent cache into memory.

void push (struct file_sha1 **stack, struct file_sha1 *record)

Push a record onto a stack (either waiting_for_sha1_computation or waiting_for_library_build_complete).

void free_cell (struct file_sha1 *cell)

Free a working cell.

void sha1_computation_context_free (gpointer u)

void put_sha1_back_into_share_library (struct shared_file *sf, const char *file_name, const char *digest)

When SHA1 is computed, and we know what struct share it's related to, we call this function to update set the share SHA1 value.

void try_to_put_sha1_back_into_share_library (void)

We have some SHA1 we couldn't put the values into the share library because it wasn't available.

void close_current_file (struct sha1_computation_context *ctx)

Close the file whose hash we're computing (after calculation completed) and free the associated structure.

file_sha1 * get_next_file_from_list (void)

Get the next file waiting for its hash to be computed from the queue (actually a stack).

gboolean open_next_file (struct sha1_computation_context *ctx)

Open the next file waiting for its hash to be computed.

void got_sha1_result (struct sha1_computation_context *ctx, char *digest)

Callback to be called when a computation has completed.

void sha1_timer_one_step (struct sha1_computation_context *ctx, gint ticks, gint *used)

The timer calls repeatedly this function, consuming one unit of credit every call.

bgret_t sha1_step_compute (gpointer h, gpointer u, gint ticks)

The routine doing all the work.

bgret_t sha1_step_dump (gpointer unused_h, gpointer unused_u, gint unused_ticks)

Dump SHA1 cache if it is dirty.

void queue_shared_file_for_sha1_computation (guint32 file_index, const char *file_name)

Put the file with a given file_index and file_name on the stack of the things to do.

gboolean cached_entry_up_to_date (const struct sha1_cache_entry *cache_entry, const struct shared_file *sf)

Check to see if an (in-memory) entry cache is up to date.

gboolean sha1_is_cached (const struct shared_file *sf)

External interface to check whether the sha1 for shared_file is known.

void request_sha1 (struct shared_file *sf)

External interface to call for getting the hash for a shared_file.

void huge_init (void)

Call this function at the beginning.

gboolean cache_free_entry (gpointer unused_key, gpointer v, gpointer unused_udata)

Free SHA1 cache entry.

void huge_close (void)

Call this when servent is shutdown.

gboolean huge_improbable_sha1 (const gchar *buf, size_t len)

Test whether the SHA1 in its base32/binary form is improbable.

gboolean huge_sha1_extract32 (const gchar *buf, size_t len, gchar *retval, gpointer header, gboolean check_old)

huge_sha1_extract32

void huge_collect_locations (gchar *sha1, header_t *header)

Parse the "X-Gnutella-Alternate-Location" header if present to learn about other sources for this file.

Variables

GHashTable * sha1_cache = NULL

gboolean cache_dirty = FALSE

cache_dirty means that in-core cache is different from the one on disk when TRUE.

const char sha1_persistent_cache_file_header []

Disk cache.

char * persistent_cache_file_name = NULL

gpointer sha1_task = NULL

file_sha1 * waiting_for_sha1_computation = NULL

When a hash is requested for a file and is unknown, it is first stored onto this stack, waiting to be processed.

file_sha1 * waiting_for_library_build_complete = NULL

When the hash for a file has been computed but cannot be set into the struct shared_file because the function shared_file returned SHARE_REBUILDING (for example), the corresponding struct file_hash is stored into this stack, until such time it's possible to get struct shared_file from index with shared_file.

Define Documentation

#define HASH_BLOCK_SHIFT 12 /**< Power of two of hash unit credit */

Power of two of hash unit credit.

#define HASH_BUF_SIZE 65536 /**< Size of a the reading buffer */

Size of a the reading buffer.

#define SHA1_MAGIC 0xa1a1a1a1U

Function Documentation

void add_persistent_cache_entry ( const char * filename,

filesize_t size,

time_t mtime,

const char * digest

) [static]

Add an entry to the persistent cache.

void add_volatile_cache_entry ( const char * filename,

filesize_t size,

time_t mtime,

const char * digest,

gboolean known_to_be_shared

) [static]

add_volatile_cache_entry
Add a new entry to the in-memory cache.

void cache_entry_print ( FILE * f,

const char * filename,

const gchar * digest,

filesize_t size,

time_t mtime

) [static]

gboolean cache_free_entry ( gpointer unused_key,

gpointer v,

gpointer unused_udata

) [static]

Free SHA1 cache entry.

gboolean cached_entry_up_to_date ( const struct sha1_cache_entry * cache_entry,

const struct shared_file * sf

) [static]

Check to see if an (in-memory) entry cache is up to date.

Returns:
true (in the C sense) if it is, or false otherwise.

void close_current_file ( struct sha1_computation_context * ctx ) [static]

Close the file whose hash we're computing (after calculation completed) and free the associated structure.

void copy_sha1 ( char * dest,

const char * source

) [static]

copy_sha1
Copy the ASCII representation of a SHA1 digest from source to dest.

void dump_cache ( void ) [static]

Dump the whole in-memory cache onto disk.

void dump_cache_one_entry ( gpointer unused_key,

gpointer value,

gpointer udata

) [static]

Dump one (in-memory) cache into the persistent cache.
This is a callback called by dump_cache to dump the whole in-memory cache onto disk.

void free_cell ( struct file_sha1 * cell ) [static]

Free a working cell.

struct file_sha1* get_next_file_from_list ( void ) [static]

Get the next file waiting for its hash to be computed from the queue (actually a stack).

Returns:
this file.

void got_sha1_result ( struct sha1_computation_context * ctx,

char * digest

) [static]

Callback to be called when a computation has completed.

void huge_close ( void )

Call this when servent is shutdown.

void huge_collect_locations ( gchar * sha1,

header_t * header

)

Parse the "X-Gnutella-Alternate-Location" header if present to learn about other sources for this file.

gboolean huge_improbable_sha1 ( const gchar * buf,

size_t len

)

Test whether the SHA1 in its base32/binary form is improbable.
This is used to detect "urn:sha1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" and things using the same pattern with other letters, as being rather improbable hashes.

void huge_init ( void )

Call this function at the beginning.

gboolean huge_sha1_extract32 ( const gchar * buf,

size_t len,

gchar * retval,

gpointer header,

gboolean check_old

)

huge_sha1_extract32
Validate `len' bytes starting from `buf' as a proper base32 encoding of a SHA1 hash, and write decoded value in `retval'. Also make sure that the SHA1 is not an improbable value.
`header' is the header of the packet where we found the SHA1, so that we may trace errors if needed.
When `check_old' is true, check the encoding against an earlier version of the base32 alphabet.

Returns:
TRUE if the SHA1 was valid and properly decoded, FALSE on error.

gboolean open_next_file ( struct sha1_computation_context * ctx ) [static]

Open the next file waiting for its hash to be computed.

Returns:
TRUE if open succeeded, FALSE otherwise.

void parse_and_append_cache_entry ( char * line ) [static]

This function is used to read the disk cache into memory.
It must be passed one line from the cache (ending with '
'). It performs all the syntactic processing to extract the fields from the line and calls add_volatile_cache_entry() to append the record to the in-memory cache.

void push ( struct file_sha1 ** stack,

struct file_sha1 * record

) [static]

Push a record onto a stack (either waiting_for_sha1_computation or waiting_for_library_build_complete).

void put_sha1_back_into_share_library ( struct shared_file * sf,

const char * file_name,

const char * digest

) [static]

When SHA1 is computed, and we know what struct share it's related to, we call this function to update set the share SHA1 value.

void queue_shared_file_for_sha1_computation ( guint32 file_index,

const char * file_name

) [static]

Put the file with a given file_index and file_name on the stack of the things to do.
Activate the timer if this wasn't done already.

RCSID ( "$Id:huge. c,

v 1.24 2005/09/10 08:17:28 daichik Exp $"

)

void request_sha1 ( struct shared_file * sf )

External interface to call for getting the hash for a shared_file.

void sha1_computation_context_free ( gpointer u ) [static]

gboolean sha1_is_cached ( const struct shared_file * sf )

External interface to check whether the sha1 for shared_file is known.

void sha1_read_cache ( void ) [static]

Read the whole persistent cache into memory.

bgret_t sha1_step_compute ( gpointer h,

gpointer u,

gint ticks

) [static]

The routine doing all the work.

bgret_t sha1_step_dump ( gpointer unused_h,

gpointer unused_u,

gint unused_ticks

) [static]

Dump SHA1 cache if it is dirty.

void sha1_timer_one_step ( struct sha1_computation_context * ctx,

gint ticks,

gint * used

) [static]

The timer calls repeatedly this function, consuming one unit of credit every call.

void try_to_put_sha1_back_into_share_library ( void ) [static]

We have some SHA1 we couldn't put the values into the share library because it wasn't available.
We try again. This function is called from the sha1_timer.

void update_volatile_cache ( struct sha1_cache_entry * sha1_cached_entry,

filesize_t size,

time_t mtime,

const char * digest

) [static]

update_volatile_cache
Takes an in-memory cached entry, and update its content.

Variable Documentation

gboolean cache_dirty = FALSE [static]

cache_dirty means that in-core cache is different from the one on disk when TRUE.

char* persistent_cache_file_name = NULL [static]

GHashTable* sha1_cache = NULL [static]

const char sha1_persistent_cache_file_header[] [static]

Initial value:
"#\n" "# gtk-gnutella SHA1 cache file.\n" "# This file is automatically generated.\n" "# Format is: SHA1 digest<TAB>file_size<TAB>file_mtime<TAB>file_name\n" "# Comment lines start with a sharp (#)\n" "#\n" "\n"
Disk cache.

gpointer sha1_task = NULL [static]

struct file_sha1* waiting_for_library_build_complete = NULL [static]

When the hash for a file has been computed but cannot be set into the struct shared_file because the function shared_file returned SHARE_REBUILDING (for example), the corresponding struct file_hash is stored into this stack, until such time it's possible to get struct shared_file from index with shared_file.

struct file_sha1* waiting_for_sha1_computation = NULL [static]

When a hash is requested for a file and is unknown, it is first stored onto this stack, waiting to be processed.

Generated on Sun Feb 12 10:50:03 2006 for Gtk-Gnutella by

1.3.6


Data Structures
struct	sha1_cache_entry
	There's an in-core cache (the GHashTable sha1_cache), and a persistent copy (normally in ~/.gtk-gnutella/sha1_cache). More...
struct	file_sha1
	This is a file waiting either for the digest to be computer, or when computed to be retrofit into the share record. More...
struct	sha1_computation_context
Defines
#define	HASH_BLOCK_SHIFT 12 /*< Power of two of hash unit credit /
	Power of two of hash unit credit.
#define	HASH_BUF_SIZE 65536 /*< Size of a the reading buffer /
	Size of a the reading buffer.
#define	SHA1_MAGIC 0xa1a1a1a1U
Functions
	RCSID ("$Id:huge.c, v 1.24 2005/09/10 08:17:28 daichik Exp $")
void	copy_sha1 (char dest, const char source)
	copy_sha1
void	update_volatile_cache (struct sha1_cache_entry sha1_cached_entry, filesize_t size, time_t mtime, const char digest)
	update_volatile_cache
void	add_volatile_cache_entry (const char filename, filesize_t size, time_t mtime, const char digest, gboolean known_to_be_shared)
	add_volatile_cache_entry
void	cache_entry_print (FILE f, const char filename, const gchar *digest, filesize_t size, time_t mtime)
void	add_persistent_cache_entry (const char filename, filesize_t size, time_t mtime, const char digest)
	Add an entry to the persistent cache.
void	dump_cache_one_entry (gpointer unused_key, gpointer value, gpointer udata)
	Dump one (in-memory) cache into the persistent cache.
void	dump_cache (void)
	Dump the whole in-memory cache onto disk.
void	parse_and_append_cache_entry (char *line)
	This function is used to read the disk cache into memory.
void	sha1_read_cache (void)
	Read the whole persistent cache into memory.
void	push (struct file_sha1 *stack, struct file_sha1 record)
	Push a record onto a stack (either waiting_for_sha1_computation or waiting_for_library_build_complete).
void	free_cell (struct file_sha1 *cell)
	Free a working cell.
void	sha1_computation_context_free (gpointer u)
void	put_sha1_back_into_share_library (struct shared_file sf, const char file_name, const char *digest)
	When SHA1 is computed, and we know what struct share it's related to, we call this function to update set the share SHA1 value.
void	try_to_put_sha1_back_into_share_library (void)
	We have some SHA1 we couldn't put the values into the share library because it wasn't available.
void	close_current_file (struct sha1_computation_context *ctx)
	Close the file whose hash we're computing (after calculation completed) and free the associated structure.
file_sha1 *	get_next_file_from_list (void)
	Get the next file waiting for its hash to be computed from the queue (actually a stack).
gboolean	open_next_file (struct sha1_computation_context *ctx)
	Open the next file waiting for its hash to be computed.
void	got_sha1_result (struct sha1_computation_context ctx, char digest)
	Callback to be called when a computation has completed.
void	sha1_timer_one_step (struct sha1_computation_context ctx, gint ticks, gint used)
	The timer calls repeatedly this function, consuming one unit of credit every call.
bgret_t	sha1_step_compute (gpointer h, gpointer u, gint ticks)
	The routine doing all the work.
bgret_t	sha1_step_dump (gpointer unused_h, gpointer unused_u, gint unused_ticks)
	Dump SHA1 cache if it is dirty.
void	queue_shared_file_for_sha1_computation (guint32 file_index, const char *file_name)
	Put the file with a given file_index and file_name on the stack of the things to do.
gboolean	cached_entry_up_to_date (const struct sha1_cache_entry cache_entry, const struct shared_file sf)
	Check to see if an (in-memory) entry cache is up to date.
gboolean	sha1_is_cached (const struct shared_file *sf)
	External interface to check whether the sha1 for shared_file is known.
void	request_sha1 (struct shared_file *sf)
	External interface to call for getting the hash for a shared_file.
void	huge_init (void)
	Call this function at the beginning.
gboolean	cache_free_entry (gpointer unused_key, gpointer v, gpointer unused_udata)
	Free SHA1 cache entry.
void	huge_close (void)
	Call this when servent is shutdown.
gboolean	huge_improbable_sha1 (const gchar *buf, size_t len)
	Test whether the SHA1 in its base32/binary form is improbable.
gboolean	huge_sha1_extract32 (const gchar buf, size_t len, gchar retval, gpointer header, gboolean check_old)
	huge_sha1_extract32
void	huge_collect_locations (gchar sha1, header_t header)
	Parse the "X-Gnutella-Alternate-Location" header if present to learn about other sources for this file.
Variables
GHashTable *	sha1_cache = NULL
gboolean	cache_dirty = FALSE
	cache_dirty means that in-core cache is different from the one on disk when TRUE.
const char	sha1_persistent_cache_file_header []
	Disk cache.
char *	persistent_cache_file_name = NULL
gpointer	sha1_task = NULL
file_sha1 *	waiting_for_sha1_computation = NULL
	When a hash is requested for a file and is unknown, it is first stored onto this stack, waiting to be processed.
file_sha1 *	waiting_for_library_build_complete = NULL
	When the hash for a file has been computed but cannot be set into the struct shared_file because the function shared_file returned SHARE_REBUILDING (for example), the corresponding struct file_hash is stored into this stack, until such time it's possible to get struct shared_file from index with shared_file.