From: <tb...@us...> - 2011-05-26 07:10:42
|
Revision: 5355 http://gfarm.svn.sourceforge.net/gfarm/?rev=5355&view=rev Author: tbaba Date: 2011-05-26 07:10:33 +0000 (Thu, 26 May 2011) Log Message: ----------- add mdcluster and the function of dynamic restructure of sync/async group. * now mdhost is stored in db. * add a mutex to mdhost and remove directly access to the abstract_host's mutex. Modified Paths: -------------- gfarm_v2/trunk/gftool/gfjournal/gfjournal.c gfarm_v2/trunk/server/gfmd/Makefile gfarm_v2/trunk/server/gfmd/abstract_host.c gfarm_v2/trunk/server/gfmd/abstract_host.h gfarm_v2/trunk/server/gfmd/back_channel.c gfarm_v2/trunk/server/gfmd/db_access.c gfarm_v2/trunk/server/gfmd/db_access.h gfarm_v2/trunk/server/gfmd/db_common.c gfarm_v2/trunk/server/gfmd/db_journal.c gfarm_v2/trunk/server/gfmd/db_journal.h gfarm_v2/trunk/server/gfmd/db_journal_apply.c gfarm_v2/trunk/server/gfmd/db_ldap.c gfarm_v2/trunk/server/gfmd/db_none.c gfarm_v2/trunk/server/gfmd/db_ops.h gfarm_v2/trunk/server/gfmd/db_pgsql.c gfarm_v2/trunk/server/gfmd/gfmd.c gfarm_v2/trunk/server/gfmd/gfmd_channel.c gfarm_v2/trunk/server/gfmd/gfmd_channel.h gfarm_v2/trunk/server/gfmd/host.c gfarm_v2/trunk/server/gfmd/journal_file.c gfarm_v2/trunk/server/gfmd/journal_file.h gfarm_v2/trunk/server/gfmd/mdhost.c gfarm_v2/trunk/server/gfmd/mdhost.h Added Paths: ----------- gfarm_v2/trunk/server/gfmd/mdcluster.c gfarm_v2/trunk/server/gfmd/mdcluster.h Modified: gfarm_v2/trunk/gftool/gfjournal/gfjournal.c =================================================================== --- gfarm_v2/trunk/gftool/gfjournal/gfjournal.c 2011-05-26 06:37:17 UTC (rev 5354) +++ gfarm_v2/trunk/gftool/gfjournal/gfjournal.c 2011-05-26 07:10:33 UTC (rev 5355) @@ -20,6 +20,7 @@ #include "config.h" #include "metadb_common.h" +#include "metadb_server.h" #include "xattr_info.h" #include "quota_info.h" #include "quota.h" Modified: gfarm_v2/trunk/server/gfmd/Makefile =================================================================== --- gfarm_v2/trunk/server/gfmd/Makefile 2011-05-26 06:37:17 UTC (rev 5354) +++ gfarm_v2/trunk/server/gfmd/Makefile 2011-05-26 07:10:33 UTC (rev 5355) @@ -33,7 +33,7 @@ subr.c rpcsubr.c user.c group.c host.c peer.c dead_file_copy.c \ process.c job.c abstract_host.c \ dir.c inode.c fs.c back_channel.c acl.c journal_file.c \ - mdhost.c gfmd_channel.c \ + mdhost.c gfmd_channel.c mdcluster.c \ db_access.c db_common.c db_none.c quota.c xattr.c \ db_journal.c db_journal_apply.c \ $(ldap_srcs) $(postgresql_srcs) $(optional_srcs) @@ -41,7 +41,7 @@ subr.o rpcsubr.o user.o group.o host.o peer.o dead_file_copy.o \ process.o job.o abstract_host.o \ dir.o inode.o fs.o back_channel.o acl.o journal_file.o \ - mdhost.o gfmd_channel.o \ + mdhost.o gfmd_channel.o mdcluster.o \ db_access.o db_common.o db_none.o quota.o xattr.o \ db_journal.o db_journal_apply.o \ $(ldap_objs) $(postgresql_objs) $(optional_objs) @@ -68,10 +68,10 @@ $(GFARMLIB_SRCDIR)/gfm_proto.h \ $(GFARMLIB_SRCDIR)/gfj_client.h \ $(GFARMLIB_SRCDIR)/timespec.h \ - thrpool.h subr.h rpcsubr.h callout.h acl.h \ + thrpool.h subr.h rpcsubr.h callout.h acl.h db_access.h \ user.h group.h host.h peer.h dead_file_copy.h process.h job.h \ dir.h inode.h fs.h back_channel.h protocol_state.h quota.h xattr.h \ journal_file.h db_journal.h db_journal_apply.h abstract_host.h \ - gfmd_channel.h mdhost.h + gfmd_channel.h mdhost.h mdcluster.h include $(optional_rule) Modified: gfarm_v2/trunk/server/gfmd/abstract_host.c =================================================================== --- gfarm_v2/trunk/server/gfmd/abstract_host.c 2011-05-26 06:37:17 UTC (rev 5354) +++ gfarm_v2/trunk/server/gfmd/abstract_host.c 2011-05-26 07:10:33 UTC (rev 5355) @@ -43,6 +43,8 @@ } } +#define ABSTRACT_HOST_MUTEX_DIAG "abstract_host_mutex" + void abstract_host_init(struct abstract_host *h, struct abstract_host_ops *ops, const char *diag) @@ -58,7 +60,7 @@ gfarm_cond_init(&h->ready_to_send, diag, "ready_to_send"); gfarm_cond_init(&h->ready_to_receive, diag, "ready_to_receive"); - gfarm_mutex_init(&h->back_channel_mutex, diag, "back_channel"); + gfarm_mutex_init(&h->mutex, diag, ABSTRACT_HOST_MUTEX_DIAG); } int @@ -91,26 +93,35 @@ return (h->invalid == 0); } +static void +abstract_host_mutex_lock(struct abstract_host *h, const char *diag) +{ + gfarm_mutex_lock(&h->mutex, diag, ABSTRACT_HOST_MUTEX_DIAG); +} + +static void +abstract_host_mutex_unlock(struct abstract_host *h, const char *diag) +{ + gfarm_mutex_unlock(&h->mutex, diag, ABSTRACT_HOST_MUTEX_DIAG); +} + int -abstract_host_is_valid(struct abstract_host *h, const char *back_channel_diag) +abstract_host_is_valid(struct abstract_host *h, const char *diag) { int valid; - static const char diag[] = "abstract_host_is_valid"; - gfarm_mutex_lock(&h->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_lock(h, diag); valid = abstract_host_is_valid_unlocked(h); - gfarm_mutex_unlock(&h->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_unlock(h, diag); return (valid); } void -abstract_host_activate(struct abstract_host *h, const char *back_channel_diag) +abstract_host_activate(struct abstract_host *h, const char *diag) { - static const char diag[] = "abstract_host_activate"; - - gfarm_mutex_lock(&h->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_lock(h, diag); h->is_active = 1; - gfarm_mutex_unlock(&h->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_unlock(h, diag); } struct host * @@ -145,37 +156,23 @@ /* * PREREQUISITE: nothing - * LOCKS: host::back_channel_mutex + * LOCKS: host::channel_mutex * SLEEPS: no */ int -abstract_host_is_up(struct abstract_host *h, const char *back_channel_diag) +abstract_host_is_up(struct abstract_host *h) { int up; static const char diag[] = "abstract_host_is_up"; - gfarm_mutex_lock(&h->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_lock(h, diag); up = abstract_host_is_up_unlocked(h); - gfarm_mutex_unlock(&h->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_unlock(h, diag); return (up); } -void -abstract_host_channel_mutex_lock(struct abstract_host *h, - const char *diag, const char *back_channel_diag) -{ - gfarm_mutex_lock(&h->back_channel_mutex, diag, back_channel_diag); -} - -void -abstract_host_channel_mutex_unlock(struct abstract_host *h, - const char *diag, const char *back_channel_diag) -{ - gfarm_mutex_unlock(&h->back_channel_mutex, diag, back_channel_diag); -} - /* - * PREREQUISITE: host::back_channel_mutex + * PREREQUISITE: host::channel_mutex * LOCKS: nothing * SLEEPS: no */ @@ -201,14 +198,11 @@ } static void -abstract_host_peer_unbusy(struct abstract_host *host, - const char *back_channel_diag) +abstract_host_peer_unbusy(struct abstract_host *host, const char *diag) { - static const char diag[] = "abstract_host_peer_unbusy"; - - gfarm_mutex_lock(&host->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_lock(host, diag); host->busy_time = 0; - gfarm_mutex_unlock(&host->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_unlock(host, diag); } struct peer * @@ -218,25 +212,23 @@ } struct peer * -abstract_host_get_peer(struct abstract_host *h, const char *back_channel_diag) +abstract_host_get_peer(struct abstract_host *h, const char *diag) { struct peer *peer; - static const char diag[] = "abstract_host_get_peer"; - gfarm_mutex_lock(&h->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_lock(h, diag); peer = h->peer; - gfarm_mutex_unlock(&h->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_unlock(h, diag); return (peer); } gfarm_error_t abstract_host_sender_trylock(struct abstract_host *host, struct peer **peerp, - const char *back_channel_diag) + const char *diag) { gfarm_error_t e; - static const char diag[] = "abstract_host_sender_trylock"; - gfarm_mutex_lock(&host->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_lock(host, diag); if (!abstract_host_is_up_unlocked(host)) { e = GFARM_ERR_CONNECTION_ABORTED; @@ -250,20 +242,19 @@ e = GFARM_ERR_DEVICE_BUSY; } - gfarm_mutex_unlock(&host->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_unlock(host, diag); return (e); } static gfarm_error_t abstract_host_sender_lock(struct abstract_host *host, struct peer **peerp, - const char *back_channel_diag) + const char *diag) { gfarm_error_t e; struct peer *peer0; - static const char diag[] = "abstract_host_sender_lock"; - gfarm_mutex_lock(&host->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_lock(host, diag); for (;;) { if (!abstract_host_is_up_unlocked(host)) { @@ -279,7 +270,7 @@ break; } peer0 = host->peer; - gfarm_cond_wait(&host->ready_to_send, &host->back_channel_mutex, + gfarm_cond_wait(&host->ready_to_send, &host->mutex, diag, "ready_to_send"); if (host->peer != peer0) { e = GFARM_ERR_CONNECTION_ABORTED; @@ -287,19 +278,17 @@ } } - gfarm_mutex_unlock(&host->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_unlock(host, diag); return (e); } static void abstract_host_sender_unlock(struct abstract_host *host, struct peer *peer, - const char *back_channel_diag) + const char *diag) { - static const char diag[] = "abstract_host_sender_unlock"; + abstract_host_mutex_lock(host, diag); - gfarm_mutex_lock(&host->back_channel_mutex, diag, back_channel_diag); - if (peer == host->peer) { host->can_send = 1; host->busy_time = 0; @@ -307,18 +296,17 @@ peer_del_ref(peer); gfarm_cond_signal(&host->ready_to_send, diag, "ready_to_send"); - gfarm_mutex_unlock(&host->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_unlock(host, diag); } static gfarm_error_t abstract_host_receiver_lock(struct abstract_host *host, struct peer **peerp, - const char *back_channel_diag) + const char *diag) { gfarm_error_t e; struct peer *peer0; - static const char diag[] = "abstract_host_receiver_lock"; - gfarm_mutex_lock(&host->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_lock(host, diag); for (;;) { if (!abstract_host_is_up_unlocked(host)) { @@ -339,14 +327,14 @@ "maybe %s restarted?", peer_get_service_name(peer0)); gfarm_cond_wait(&host->ready_to_receive, - &host->back_channel_mutex, diag, "ready_to_receive"); + &host->mutex, diag, "ready_to_receive"); if (host->peer != peer0) { e = GFARM_ERR_CONNECTION_ABORTED; break; } } - gfarm_mutex_unlock(&host->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_unlock(host, diag); return (e); } @@ -355,32 +343,30 @@ abstract_host_receiver_unlock(struct abstract_host *host, struct peer *peer) { static const char diag[] = "abstract_host_receiver_unlock"; - const char *back_channel_diag = back_channel_type_name(peer); - gfarm_mutex_lock(&host->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_lock(host, diag); if (peer == host->peer) host->can_receive = 1; peer_del_ref(peer); gfarm_cond_signal(&host->ready_to_receive, diag, "ready_to_receive"); - gfarm_mutex_unlock(&host->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_unlock(host, diag); } /* * PREREQUISITE: giant_lock - * LOCKS: host::back_channel_mutex, dfc_allq.mutex, removal_pendingq.mutex + * LOCKS: host::channel_mutex, dfc_allq.mutex, removal_pendingq.mutex * SLEEPS: maybe (see the comment of dead_file_copy_host_becomes_up()) - * but host::back_channel_mutex, dfc_allq.mutex and removal_pendingq.mutex + * but host::channel_mutex, dfc_allq.mutex and removal_pendingq.mutex * won't be blocked while sleeping. */ void abstract_host_set_peer(struct abstract_host *h, struct peer *p, int version) { static const char diag[] = "abstract_host_set_peer"; - const char *back_channel_diag = back_channel_type_name(p); - gfarm_mutex_lock(&h->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_lock(h, diag); h->can_send = 1; h->can_receive = 1; @@ -390,13 +376,13 @@ h->busy_time = 0; h->ops->set_peer_locked(h, p); - gfarm_mutex_unlock(&h->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_unlock(h, diag); h->ops->set_peer_unlocked(h, p); } /* - * PREREQUISITE: host::back_channel_mutex + * PREREQUISITE: host::channel_mutex * LOCKS: nothing * SLEEPS: no * @@ -412,7 +398,7 @@ } /* - * PREREQUISITE: host::back_channel_mutex + * PREREQUISITE: host::channel_mutex * LOCKS: removal_pendingq.mutex, host_busyq.mutex * SLEEPS: no */ @@ -436,9 +422,8 @@ void *closure; struct peer *hpeer; static const char diag[] = "abstract_host_disconnect_request"; - const char *back_channel_diag = back_channel_type_name(peer); - gfarm_mutex_lock(&h->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_lock(h, diag); hpeer = h->peer; if (h->is_active && (peer == hpeer || peer == NULL)) { @@ -449,27 +434,25 @@ disabled = 1; } - gfarm_mutex_unlock(&h->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_unlock(h, diag); if (disabled) h->ops->disabled(h, hpeer, closure); } static void -abstract_host_peer_busy(struct abstract_host *host, - const char *back_channel_diag) +abstract_host_peer_busy(struct abstract_host *host, const char *diag) { struct peer *unresponsive_peer = NULL; - static const char diag[] = "abstract_host_peer_busy"; - gfarm_mutex_lock(&host->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_lock(host, diag); if (!host->is_active || host->invalid) ; else if (host->busy_time == 0) host->busy_time = time(NULL); else if (abstract_host_is_unresponsive(host, time(NULL), diag)) unresponsive_peer = host->peer; - gfarm_mutex_unlock(&host->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_unlock(host, diag); if (unresponsive_peer != NULL) { gflog_error(GFARM_MSG_UNFIXED, @@ -482,20 +465,19 @@ int abstract_host_check_busy(struct abstract_host *host, gfarm_int64_t now, - const char *back_channel_diag) + const char *diag) { int busy = 0; struct peer *unresponsive_peer = NULL; - static const char diag[] = "abstract_host_check_busy"; - gfarm_mutex_lock(&host->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_lock(host, diag); if (!host->is_active || host->invalid) busy = 1; else if (abstract_host_is_unresponsive(host, now, diag)) unresponsive_peer = host->peer; - gfarm_mutex_unlock(&host->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_unlock(host, diag); if (unresponsive_peer != NULL) { gflog_error(GFARM_MSG_UNFIXED, @@ -511,19 +493,18 @@ /* giant_lock should be held before calling this */ void abstract_host_disconnect(struct abstract_host *h, struct peer *peer, - const char *back_channel_diag) + const char *diag) { #if 0 /* * commented out, - * not to sleep while holding host::back_channel_mutex + * not to sleep while holding host::channel_mutex */ int disabled = 0; void *closure; - static const char diag[] = "abstract_host_disconnect"; - gfarm_mutex_lock(&h->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_lock(h, diag); if (h->is_active && (peer == h->peer || peer == NULL)) { peer_record_protocol_error(h->peer); @@ -542,7 +523,7 @@ disabled = 1; } - gfarm_mutex_unlock(&h->back_channel_mutex, diag, back_channel_diag); + abstract_host_mutex_unlock(h, diag); if (disabled) h->ops->disabled(h, closure); @@ -767,8 +748,7 @@ gfarm_int32_t (*)(void *, void *, size_t), void (*)(void *, void *), void *), #endif - gfarm_int32_t command, const char *format, va_list *app, - const char *back_channel_diag) + gfarm_int32_t command, const char *format, va_list * app) { gfarm_error_t e; struct peer *peer; @@ -780,14 +760,14 @@ "%s: <%s> channel sending request(%d)", abstract_host_get_name(host), diag, command); - e = abstract_host_sender_trylock(host, &peer, back_channel_diag); + e = abstract_host_sender_trylock(host, &peer, diag); if (e != GFARM_ERR_NO_ERROR) { if (e == GFARM_ERR_DEVICE_BUSY) { gflog_debug(GFARM_MSG_UNFIXED, "%s(%s) channel (command %d) request: " "sending busy", abstract_host_get_name(host), diag, command); - abstract_host_peer_busy(host, back_channel_diag); + abstract_host_peer_busy(host, diag); } else /* host_disconnect_request() is already called */ gfm_server_channel_already_disconnected_message(host, diag, "request", "sending busy"); @@ -798,13 +778,13 @@ abstract_host_sender_unlock(host, peer, back_channel_type_name(peer0)); gflog_debug(GFARM_MSG_UNFIXED, - "%s(%s) %s (command %d) request: " + "(%s) %s (command %d) request: " "%s was reconnected", - back_channel_diag, abstract_host_get_name(host), diag, + abstract_host_get_name(host), diag, command, peer_get_service_name(peer0)); return (GFARM_ERR_CONNECTION_ABORTED); } - abstract_host_peer_unbusy(host, back_channel_diag); + abstract_host_peer_unbusy(host, diag); async = peer_get_async(peer); server = peer_get_conn(peer); @@ -832,12 +812,12 @@ if (e != GFARM_ERR_NO_ERROR) { /* must be IS_CONNECTION_ERROR(e) */ gfm_server_channel_disconnect_request(host, peer, diag, "request", gfarm_error_string(e)); - abstract_host_sender_unlock(host, peer, back_channel_diag); + abstract_host_sender_unlock(host, peer, diag); return (e); } if (async != NULL) /* is asynchronous mode? */ - abstract_host_sender_unlock(host, peer, back_channel_diag); + abstract_host_sender_unlock(host, peer, diag); return (GFARM_ERR_NO_ERROR); } @@ -868,8 +848,7 @@ gfarm_error_t gfm_server_channel_vput_reply(struct abstract_host *host, struct peer *peer0, gfp_xdr_xid_t xid, - const char *diag, gfarm_error_t errcode, char *format, va_list *app, - const char *back_channel_diag) + const char *diag, gfarm_error_t errcode, char *format, va_list *app) { gfarm_error_t e; struct peer *peer; @@ -877,16 +856,15 @@ if (debug_mode) gflog_info(GFARM_MSG_UNFIXED, - "%s: <%s> %s sending reply: %d", - abstract_host_get_name(host), diag, - back_channel_diag, (int)errcode); + "%s: <%s> sending reply: %d", + abstract_host_get_name(host), diag, (int)errcode); /* * Since this is a reply, the peer is probably living, * thus, not using peer_sender_trylock() is mostly ok. */ - if ((e = abstract_host_sender_lock(host, &peer, - back_channel_diag)) != GFARM_ERR_NO_ERROR) + if ((e = abstract_host_sender_lock(host, &peer, diag)) + != GFARM_ERR_NO_ERROR) return (e); if (peer != peer0) return (GFARM_ERR_CONNECTION_ABORTED); @@ -895,7 +873,7 @@ if (e == GFARM_ERR_NO_ERROR) e = gfp_xdr_flush(client); - abstract_host_sender_unlock(host, peer, back_channel_diag); + abstract_host_sender_unlock(host, peer, diag); if (e != GFARM_ERR_NO_ERROR) gflog_error(GFARM_MSG_UNFIXED, Modified: gfarm_v2/trunk/server/gfmd/abstract_host.h =================================================================== --- gfarm_v2/trunk/server/gfmd/abstract_host.h 2011-05-26 06:37:17 UTC (rev 5354) +++ gfarm_v2/trunk/server/gfmd/abstract_host.h 2011-05-26 07:10:33 UTC (rev 5355) @@ -22,10 +22,10 @@ int invalid; /* set when deleted */ + pthread_mutex_t mutex; /* - * resources which are protected by the host::back_channel_mutex + * resources which are protected by the abstrac_host::mutex */ - pthread_mutex_t back_channel_mutex; pthread_cond_t ready_to_send, ready_to_receive; int can_send, can_receive; @@ -50,13 +50,9 @@ int abstract_host_is_valid(struct abstract_host *, const char *); void abstract_host_activate(struct abstract_host *, const char *); int abstract_host_is_up_unlocked(struct abstract_host *); -int abstract_host_is_up(struct abstract_host *, const char *); +int abstract_host_is_up(struct abstract_host *); const char *abstract_host_get_name(struct abstract_host *); int abstract_host_get_port(struct abstract_host *); -void abstract_host_channel_mutex_lock(struct abstract_host *, - const char *, const char *); -void abstract_host_channel_mutex_unlock(struct abstract_host *, - const char *, const char *); int abstract_host_check_busy(struct abstract_host *, gfarm_int64_t, const char *); struct peer *abstract_host_get_peer_unlocked(struct abstract_host *); @@ -83,7 +79,7 @@ const char *, const char *, va_list *); gfarm_error_t gfm_server_channel_vput_reply(struct abstract_host *, struct peer *, gfp_xdr_xid_t, const char *, gfarm_error_t, - char *, va_list *, const char *); + char *, va_list *); gfarm_error_t gfm_client_channel_vsend_request(struct abstract_host *, struct peer *, const char *, gfarm_int32_t (*)(void *, void *, size_t), @@ -93,7 +89,7 @@ gfarm_int32_t (*)(void *, void *, size_t), void (*)(void *, void *), void *), #endif - gfarm_int32_t, const char *, va_list *, const char *); + gfarm_int32_t, const char *, va_list *); gfarm_error_t gfm_client_channel_vrecv_result(struct peer *, struct abstract_host *, size_t, const char *, const char **, gfarm_error_t *, va_list *); Modified: gfarm_v2/trunk/server/gfmd/back_channel.c =================================================================== --- gfarm_v2/trunk/server/gfmd/back_channel.c 2011-05-26 06:37:17 UTC (rev 5354) +++ gfarm_v2/trunk/server/gfmd/back_channel.c 2011-05-26 07:10:33 UTC (rev 5355) @@ -37,7 +37,7 @@ static struct thread_pool *back_channel_recv_thread_pool; static struct thread_pool *back_channel_send_thread_pool; -#define BACK_CHANNEL_DIAG "back_channel" +static const char BACK_CHANNEL_DIAG[] = "back_channel"; /* * responsibility to call host_disconnect(): @@ -95,7 +95,7 @@ va_start(ap, format); e = gfm_server_channel_vput_reply( host_to_abstract_host(host), peer, xid, diag, - errcode, format, &ap, BACK_CHANNEL_DIAG); + errcode, format, &ap); va_end(ap); return (e); @@ -188,7 +188,7 @@ #ifdef COMPAT_GFARM_2_3 host_set_callback, #endif - command, format, &ap, BACK_CHANNEL_DIAG); + command, format, &ap); va_end(ap); return (e); } Modified: gfarm_v2/trunk/server/gfmd/db_access.c =================================================================== --- gfarm_v2/trunk/server/gfmd/db_access.c 2011-05-26 06:37:17 UTC (rev 5354) +++ gfarm_v2/trunk/server/gfmd/db_access.c 2011-05-26 07:10:33 UTC (rev 5355) @@ -18,6 +18,7 @@ #include "subr.h" #include "quota_info.h" +#include "metadb_server.h" #include "quota.h" #include "mdhost.h" #include "db_access.h" @@ -1753,4 +1754,89 @@ return ((*ops->seqnum_load)(closure, callback)); } +void * +db_mdhost_dup(const struct gfarm_metadb_server *ms, size_t size) +{ + struct gfarm_metadb_server *r; + size_t nsize = strlen(ms->name) + 1; + size_t csize = ms->clustername ? strlen(ms->clustername) + 1 : 0; + size_t sz; + int overflow = 0; + +#ifdef __GNUC__ /* workaround gcc warning: might be used uninitialized */ + r = NULL; +#endif + sz = gfarm_size_add(&overflow, size, nsize + csize); + if (!overflow) + r = malloc(sz); + if (overflow || r == NULL) { + gflog_debug(GFARM_MSG_UNFIXED, + "allocation of 'gfarm_metadb_server' failed or overflow"); + return (NULL); + } + r->name = (char *)r + size; + r->clustername = ms->clustername ? r->name + nsize : NULL; + + strcpy(r->name, ms->name); + if (ms->clustername) + strcpy(r->clustername, ms->clustername); + r->port = ms->port; + r->flags = ms->flags; + return (r); +} + +gfarm_error_t +db_mdhost_add(const struct gfarm_metadb_server *ms) +{ + struct gfarm_metadb_server *m = db_mdhost_dup(ms, sizeof(*ms)); + + if (m == NULL) { + gflog_debug(GFARM_MSG_UNFIXED, "db_mdhost_dup() failed"); + return (GFARM_ERR_NO_MEMORY); + } + return (db_enter_sn((dbq_entry_func_t)ops->mdhost_add, m)); +} + +struct db_mdhost_modify_arg * +db_mdhost_modify_arg_alloc(const struct gfarm_metadb_server *ms, int modflags) +{ + struct db_mdhost_modify_arg *arg = db_mdhost_dup(ms, sizeof(*arg)); + + if (arg == NULL) + return (NULL); + arg->modflags = modflags; + return (arg); +} + +gfarm_error_t +db_mdhost_modify(const struct gfarm_metadb_server *ms, int modflags) +{ + struct db_mdhost_modify_arg *arg = db_mdhost_modify_arg_alloc( + ms, modflags); + + if (arg == NULL) { + gflog_debug(GFARM_MSG_UNFIXED, + "db_mdhost_modify_arg_alloc failed"); + return (GFARM_ERR_NO_MEMORY); + } + return (db_enter_sn((dbq_entry_func_t)ops->mdhost_modify, arg)); +} + +gfarm_error_t +db_mdhost_remove(const char *name) +{ + char *u = strdup_log(name, "db_mdhost_remove"); + + if (u == NULL) + return (GFARM_ERR_NO_MEMORY); + return (db_enter_sn((dbq_entry_func_t)ops->mdhost_remove, u)); +} + +gfarm_error_t +db_mdhost_load(void *closure, + void (*callback)(void *, struct gfarm_metadb_server *)) +{ + return ((*ops->mdhost_load)(closure, callback)); +} + #endif /* ENABLE_METADATA_REPLICATION */ Modified: gfarm_v2/trunk/server/gfmd/db_access.h =================================================================== --- gfarm_v2/trunk/server/gfmd/db_access.h 2011-05-26 06:37:17 UTC (rev 5354) +++ gfarm_v2/trunk/server/gfmd/db_access.h 2011-05-26 07:10:33 UTC (rev 5355) @@ -117,6 +117,16 @@ gfarm_error_t db_seqnum_load(void *, void (*)(void *, struct db_seqnum_arg *)); pthread_mutex_t *get_db_access_mutex(void); + +struct gfarm_metadb_server; +gfarm_error_t db_mdhost_add(const struct gfarm_metadb_server *); +void *db_mdhost_dup(const struct gfarm_metadb_server *, size_t); +struct db_mdhost_modify_arg *db_mdhost_modify_arg_alloc( + const struct gfarm_metadb_server *, int); +gfarm_error_t db_mdhost_modify(const struct gfarm_metadb_server *, int); +gfarm_error_t db_mdhost_remove(const char *); +gfarm_error_t db_mdhost_load(void *, void (*)(void *, + struct gfarm_metadb_server *)); #endif /* allocation for storage operations arguments */ @@ -132,6 +142,7 @@ struct db_xattr_arg; struct db_quota_arg; struct db_quota_remove_arg; +struct db_mdhost_modify_arg; void *db_host_dup(const struct gfarm_host_info *, size_t); void *db_user_dup(const struct gfarm_user_info *, size_t); Modified: gfarm_v2/trunk/server/gfmd/db_common.c =================================================================== --- gfarm_v2/trunk/server/gfmd/db_common.c 2011-05-26 06:37:17 UTC (rev 5354) +++ gfarm_v2/trunk/server/gfmd/db_common.c 2011-05-26 07:10:33 UTC (rev 5355) @@ -8,6 +8,7 @@ #include <gfarm/gfarm.h> #include "metadb_common.h" +#include "metadb_server.h" #include "quota.h" #include "db_access.h" Modified: gfarm_v2/trunk/server/gfmd/db_journal.c =================================================================== --- gfarm_v2/trunk/server/gfmd/db_journal.c 2011-05-26 06:37:17 UTC (rev 5354) +++ gfarm_v2/trunk/server/gfmd/db_journal.c 2011-05-26 07:10:33 UTC (rev 5355) @@ -74,11 +74,11 @@ static pthread_cond_t journal_recvq_nonfull_cond; static pthread_cond_t journal_recvq_cancel_cond; -#define RECVQ_MUTEX_DIAG "journal_recvq_mutex" -#define SEQNUM_MUTEX_DIAG "journal_seqnum_mutex" -#define RECVQ_NONEMPTY_COND_DIAG "journal_recvq_nonempty_cond" -#define RECVQ_NONFULL_COND_DIAG "journal_recvq_nonfull_cond" -#define RECVQ_CANCEL_COND_DIAG "journal_recvq_cancel_cond" +static const char RECVQ_MUTEX_DIAG[] = "journal_recvq_mutex"; +static const char SEQNUM_MUTEX_DIAG[] = "journal_seqnum_mutex"; +static const char RECVQ_NONEMPTY_COND_DIAG[] = "journal_recvq_nonempty_cond"; +static const char RECVQ_NONFULL_COND_DIAG[] = "journal_recvq_nonfull_cond"; +static const char RECVQ_CANCEL_COND_DIAG[] = "journal_recvq_cancel_cond"; static gfarm_uint64_t journal_seqnum = JOURNAL_SEQNUM_NOT_SET; static gfarm_uint64_t journal_seqnum_pre = JOURNAL_SEQNUM_NOT_SET; @@ -102,7 +102,7 @@ } static gfarm_error_t -db_journal_initialize(int is_master) +db_journal_initialize(void) { gfarm_error_t e; char path[MAXPATHLEN + 1]; @@ -127,26 +127,6 @@ gfarm_error_string(e)); return (e); } - if (journal_seqnum == JOURNAL_SEQNUM_NOT_SET) { - if (!is_master) { - gflog_error(GFARM_MSG_UNFIXED, - "database is not replicated from master gfmd"); - return (GFARM_ERR_INTERNAL_ERROR); - } - if ((e = db_begin(diag)) == GFARM_ERR_NO_ERROR) { - if ((e = db_seqnum_add("", 1)) == GFARM_ERR_NO_ERROR) { - e = db_end(diag); - if (e == GFARM_ERR_NO_ERROR) - journal_seqnum = 1; - } - } - if (e != GFARM_ERR_NO_ERROR) { - gflog_error(GFARM_MSG_UNFIXED, - "failed to add the update sequence number to db" - " : %s", gfarm_error_string(e)); - return (e); - } - } snprintf(path, MAXPATHLEN, "%s/%010d.gmj", journal_dir, 0); #ifdef DEBUG_JOURNAL @@ -180,9 +160,9 @@ } void -db_journal_init(int is_master) +db_journal_init() { - if (db_journal_initialize(is_master) != GFARM_ERR_NO_ERROR) + if (db_journal_initialize() != GFARM_ERR_NO_ERROR) exit(1); } @@ -486,6 +466,21 @@ free(st); } +static void +db_journal_metadb_server_destroy(struct gfarm_metadb_server *ms) +{ + gfarm_metadb_server_free(ms); + free(ms); +} + +static void +db_journal_metadb_server_modify_arg_destroy( + struct db_mdhost_modify_arg *arg) +{ + gfarm_metadb_server_free(&arg->ms); + free(arg); +} + /**********************************************************/ static gfarm_error_t db_journal_write_begin0(gfarm_uint64_t); @@ -521,7 +516,7 @@ db_journal_fail_store_op(); return (e); } - if (ope == GFM_JOURNAL_END) { + if (ope == GFM_JOURNAL_END && db_journal_sync_op) { if ((e = db_journal_sync_op(seqnum)) != GFARM_ERR_NO_ERROR) gflog_error(GFARM_MSG_UNFIXED, "%s", gfarm_error_string(e)); @@ -2774,7 +2769,209 @@ } /**********************************************************/ +/* mdhost */ +#define GFM_JOURNAL_MDHOST_CORE_XDR_FMT "sisi" + +static gfarm_error_t +db_journal_write_mdhost_size_add(enum journal_operation ope, + size_t *sizep, void *arg) +{ + gfarm_error_t e; + struct gfarm_metadb_server *ms = arg; + + if ((e = gfp_xdr_send_size_add(sizep, + GFM_JOURNAL_MDHOST_CORE_XDR_FMT, + NON_NULL_STR(ms->name), + ms->port, + NON_NULL_STR(ms->clustername), + ms->flags)) != GFARM_ERR_NO_ERROR) { + GFLOG_DEBUG_WITH_OPE(GFARM_MSG_UNFIXED, + "gfp_xdr_send_size_add", e, ope); + return (e); + } + return (GFARM_ERR_NO_ERROR); +} + +static gfarm_error_t +db_journal_write_mdhost_core(enum journal_operation ope, void *arg) +{ + gfarm_error_t e; + struct gfarm_metadb_server *ms = arg; + + if ((e = gfp_xdr_send(JOURNAL_W_XDR, + GFM_JOURNAL_MDHOST_CORE_XDR_FMT, + NON_NULL_STR(ms->name), + ms->port, + NON_NULL_STR(ms->clustername), + ms->flags)) != GFARM_ERR_NO_ERROR) { + GFLOG_DEBUG_WITH_OPE(GFARM_MSG_UNFIXED, + "gfp_xdr_send", e, ope); + return (e); + } + return (GFARM_ERR_NO_ERROR); +} + +static gfarm_error_t +db_journal_read_mdhost_core(struct gfp_xdr *xdr, enum journal_operation ope, + struct gfarm_metadb_server *ms) +{ + gfarm_error_t e; + int eof; + + if ((e = gfp_xdr_recv(xdr, 1, &eof, + GFM_JOURNAL_MDHOST_CORE_XDR_FMT, + &ms->name, + &ms->port, + &ms->clustername, + &ms->flags)) != GFARM_ERR_NO_ERROR) { + GFLOG_DEBUG_WITH_OPE(GFARM_MSG_UNFIXED, + "gfp_xdr_recv", e, ope); + } + return (e); +} + +static gfarm_error_t +db_journal_write_mdhost_add(gfarm_uint64_t seqnum, + struct gfarm_metadb_server *arg) +{ + return (db_journal_write(seqnum, GFM_JOURNAL_MDHOST_ADD, arg, + db_journal_write_mdhost_size_add, + db_journal_write_mdhost_core)); +} + +static gfarm_error_t +db_journal_read_metadb_server(struct gfp_xdr *xdr, + struct gfarm_metadb_server **msp) +{ + gfarm_error_t e; + struct gfarm_metadb_server *ms; + const enum journal_operation ope = GFM_JOURNAL_MDHOST_ADD; + + GFARM_MALLOC(ms); + if (ms == NULL) { + GFLOG_DEBUG_WITH_OPE(GFARM_MSG_UNFIXED, + "GFARM_MALLOC", GFARM_ERR_NO_MEMORY, ope); + return (GFARM_ERR_NO_MEMORY); + } + memset(ms, 0, sizeof(*ms)); + if ((e = db_journal_read_mdhost_core(xdr, ope, ms)) + != GFARM_ERR_NO_ERROR) { + GFLOG_DEBUG_WITH_OPE(GFARM_MSG_UNFIXED, + "db_journal_read_mdhost_core", e, ope); + } + if (e == GFARM_ERR_NO_ERROR) + *msp = ms; + else { + db_journal_metadb_server_destroy(ms); + *msp = NULL; + } + return (e); +} + +static gfarm_error_t +db_journal_write_mdhost_modify_size_add(enum journal_operation ope, + size_t *sizep, void *arg) +{ + gfarm_error_t e; + struct db_mdhost_modify_arg *m = arg; + struct gfarm_metadb_server *ms = &m->ms; + + if ((e = db_journal_write_mdhost_size_add(ope, sizep, ms)) + != GFARM_ERR_NO_ERROR) { + GFLOG_DEBUG_WITH_OPE(GFARM_MSG_UNFIXED, + "db_journal_write_mdhost_size_add", e, ope); + return (e); + } + if ((e = gfp_xdr_send_size_add(sizep, "i", + m->modflags)) != GFARM_ERR_NO_ERROR) { + GFLOG_DEBUG_WITH_OPE(GFARM_MSG_UNFIXED, + "gfp_xdr_send_size_add", e, ope); + return (e); + } + return (GFARM_ERR_NO_ERROR); +} + +static gfarm_error_t +db_journal_write_mdhost_modify_core(enum journal_operation ope, void *arg) +{ + gfarm_error_t e; + struct db_mdhost_modify_arg *m = arg; + struct gfarm_metadb_server *ms = &m->ms; + + if ((e = db_journal_write_mdhost_core(ope, ms)) + != GFARM_ERR_NO_ERROR) { + GFLOG_DEBUG_WITH_OPE(GFARM_MSG_UNFIXED, + "db_journal_write_mdhost_core", e, ope); + return (e); + } + if ((e = gfp_xdr_send(JOURNAL_W_XDR, "i", + m->modflags)) != GFARM_ERR_NO_ERROR) { + GFLOG_DEBUG_WITH_OPE(GFARM_MSG_UNFIXED, + "gfp_xdr_send", e, ope); + return (e); + } + return (GFARM_ERR_NO_ERROR); +} + +static gfarm_error_t +db_journal_write_mdhost_modify(gfarm_uint64_t seqnum, + struct db_mdhost_modify_arg *arg) +{ + return (db_journal_write(seqnum, GFM_JOURNAL_MDHOST_MODIFY, arg, + db_journal_write_mdhost_modify_size_add, + db_journal_write_mdhost_modify_core)); +} + +static gfarm_error_t +db_journal_read_mdhost_modify(struct gfp_xdr *xdr, + struct db_mdhost_modify_arg **argp) +{ + gfarm_error_t e; + struct db_mdhost_modify_arg *arg; + struct gfarm_metadb_server *ms; + int eof; + const enum journal_operation ope = GFM_JOURNAL_MDHOST_MODIFY; + + GFARM_MALLOC(arg); + if (arg == NULL) { + GFLOG_DEBUG_WITH_OPE(GFARM_MSG_UNFIXED, + "GFARM_MALLOC", GFARM_ERR_NO_MEMORY, ope); + return (GFARM_ERR_NO_MEMORY); + } + memset(arg, 0, sizeof(*arg)); + ms = &arg->ms; + if ((e = db_journal_read_mdhost_core(xdr, ope, ms)) + != GFARM_ERR_NO_ERROR) { + GFLOG_DEBUG_WITH_OPE(GFARM_MSG_UNFIXED, + "db_journal_read_mdhost_core", e, ope); + goto end; + } + if ((e = gfp_xdr_recv(xdr, 1, &eof, "i", + &arg->modflags)) != GFARM_ERR_NO_ERROR) { + GFLOG_DEBUG_WITH_OPE(GFARM_MSG_UNFIXED, + "gfp_xdr_send", e, ope); + goto end; + } +end: + if (e == GFARM_ERR_NO_ERROR) + *argp = arg; + else { + db_journal_metadb_server_modify_arg_destroy(arg); + *argp = NULL; + } + return (e); +} + +static gfarm_error_t +db_journal_write_mdhost_remove(gfarm_uint64_t seqnum, char *name) +{ + return (db_journal_write_string( + seqnum, GFM_JOURNAL_MDHOST_REMOVE, name)); +} + +/**********************************************************/ + static void db_journal_ops_free(void *op_arg, enum journal_operation ope, void *obj) { @@ -2837,6 +3034,12 @@ case GFM_JOURNAL_QUOTA_REMOVE: db_journal_quota_remove_arg_destroy(obj); break; + case GFM_JOURNAL_MDHOST_ADD: + db_journal_metadb_server_destroy(obj); + break; + case GFM_JOURNAL_MDHOST_MODIFY: + db_journal_metadb_server_modify_arg_destroy(obj); + break; case GFM_JOURNAL_HOST_REMOVE: /* char[] */ case GFM_JOURNAL_USER_REMOVE: /* char[] */ case GFM_JOURNAL_GROUP_REMOVE: /* char[] */ @@ -2849,6 +3052,7 @@ case GFM_JOURNAL_INODE_CTIME_MODIFY: /* db_inode_timespec_modify_arg */ case GFM_JOURNAL_INODE_CKSUM_REMOVE: /* db_inode_inum_arg */ case GFM_JOURNAL_SYMLINK_REMOVE: /* db_inode_inum_arg */ + case GFM_JOURNAL_MDHOST_REMOVE: /* char[] */ free(obj); break; default: @@ -2897,6 +3101,7 @@ case GFM_JOURNAL_HOST_REMOVE: case GFM_JOURNAL_USER_REMOVE: case GFM_JOURNAL_GROUP_REMOVE: + case GFM_JOURNAL_MDHOST_REMOVE: e = db_journal_read_string(xdr, ope, (char **)objp); break; case GFM_JOURNAL_INODE_ADD: @@ -2969,6 +3174,14 @@ e = db_journal_read_quota_remove(xdr, (struct db_quota_remove_arg **)objp); break; + case GFM_JOURNAL_MDHOST_ADD: + e = db_journal_read_metadb_server(xdr, + (struct gfarm_metadb_server **)objp); + break; + case GFM_JOURNAL_MDHOST_MODIFY: + e = db_journal_read_mdhost_modify(xdr, + (struct db_mdhost_modify_arg **)objp); + break; default: e = GFARM_ERR_INVALID_ARGUMENT; break; @@ -3074,6 +3287,12 @@ e = ops->quota_modify(seqnum, obj); break; case GFM_JOURNAL_QUOTA_REMOVE: e = ops->quota_remove(seqnum, obj); break; + case GFM_JOURNAL_MDHOST_ADD: + e = ops->mdhost_add(seqnum, obj); break; + case GFM_JOURNAL_MDHOST_MODIFY: + e = ops->mdhost_modify(seqnum, obj); break; + case GFM_JOURNAL_MDHOST_REMOVE: + e = ops->mdhost_remove(seqnum, obj); break; default: e = GFARM_ERR_INVALID_ARGUMENT; gflog_fatal(GFARM_MSG_UNFIXED, @@ -3749,6 +3968,13 @@ return (store_ops->seqnum_load(closure, callback)); } +static gfarm_error_t +db_journal_mdhost_load(void *closure, + void (*callback)(void *, struct gfarm_metadb_server *)) +{ + return (store_ops->mdhost_load(closure, callback)); +} + /**********************************************************/ struct db_ops db_journal_ops = { @@ -3824,6 +4050,11 @@ db_journal_seqnum_modify, db_journal_seqnum_remove, db_journal_seqnum_load, + + db_journal_write_mdhost_add, + db_journal_write_mdhost_modify, + db_journal_write_mdhost_remove, + db_journal_mdhost_load, }; #endif /* ENABLE_METADATA_REPLICATION */ Modified: gfarm_v2/trunk/server/gfmd/db_journal.h =================================================================== --- gfarm_v2/trunk/server/gfmd/db_journal.h 2011-05-26 06:37:17 UTC (rev 5354) +++ gfarm_v2/trunk/server/gfmd/db_journal.h 2011-05-26 07:10:33 UTC (rev 5355) @@ -32,6 +32,6 @@ void *db_journal_recvq_thread(void *); void *db_journal_apply_thread(void *); void db_journal_boot_apply(void); -void db_journal_init(int); +void db_journal_init(void); #endif Modified: gfarm_v2/trunk/server/gfmd/db_journal_apply.c =================================================================== --- gfarm_v2/trunk/server/gfmd/db_journal_apply.c 2011-05-26 06:37:17 UTC (rev 5354) +++ gfarm_v2/trunk/server/gfmd/db_journal_apply.c 2011-05-26 07:10:33 UTC (rev 5355) @@ -8,12 +8,14 @@ #include "config.h" #include "quota.h" +#include "metadb_server.h" #include "db_ops.h" #include "host.h" #include "user.h" #include "group.h" #include "inode.h" #include "dir.h" +#include "mdhost.h" #include "db_journal.h" #ifdef ENABLE_METADATA_REPLICATION @@ -725,7 +727,71 @@ } /**********************************************************/ +/* mdhost */ +static gfarm_error_t +db_journal_apply_mdhost_add(gfarm_uint64_t seqnum, + struct gfarm_metadb_server *ms) +{ + gfarm_error_t e; + + if (mdhost_lookup(ms->name)) { + e = GFARM_ERR_ALREADY_EXISTS; + gflog_error(GFARM_MSG_UNFIXED, + "seqnum=%llu hostname=%s : %s", + (unsigned long long)seqnum, ms->name, + gfarm_error_string(e)); + } else if ((e = mdhost_enter(ms, NULL)) != GFARM_ERR_NO_ERROR) { + gflog_error(GFARM_MSG_UNFIXED, + "seqnum=%llu hostname=%s : %s", + (unsigned long long)seqnum, ms->name, + gfarm_error_string(e)); + } else + memset(ms, 0, sizeof(*ms)); + return (e); +} + +static gfarm_error_t +db_journal_apply_mdhost_modify(gfarm_uint64_t seqnum, + struct db_mdhost_modify_arg *arg) +{ + gfarm_error_t e; + struct mdhost *mh; + + if ((mh = mdhost_lookup(arg->ms.name)) == NULL) { + e = GFARM_ERR_NO_SUCH_OBJECT; + gflog_error(GFARM_MSG_UNFIXED, + "seqnum=%llu hostname=%s : %s", + (unsigned long long)seqnum, arg->ms.name, + gfarm_error_string(e)); + } else if ((e = mdhost_modify_in_cache(mh, &arg->ms)) + != GFARM_ERR_NO_ERROR) { + gflog_error(GFARM_MSG_UNFIXED, + "seqnum=%llu hostname=%s : %s", + (unsigned long long)seqnum, arg->ms.name, + gfarm_error_string(e)); + } + return (e); +} + +static gfarm_error_t +db_journal_apply_mdhost_remove(gfarm_uint64_t seqnum, char *name) +{ + gfarm_error_t e; + + if (mdhost_lookup(name) == NULL) { + e = GFARM_ERR_NO_SUCH_OBJECT; + gflog_error(GFARM_MSG_UNFIXED, + "seqnum=%llu hostname=%s : %s", + (unsigned long long)seqnum, name, + gfarm_error_string(e)); + } else + e = mdhost_remove_in_cache(name); + return (e); +} + +/**********************************************************/ + const struct db_ops db_journal_apply_ops = { NULL, NULL, @@ -799,6 +865,11 @@ NULL, NULL, NULL, + + db_journal_apply_mdhost_add, + db_journal_apply_mdhost_modify, + db_journal_apply_mdhost_remove, + NULL, }; void Modified: gfarm_v2/trunk/server/gfmd/db_ldap.c =================================================================== --- gfarm_v2/trunk/server/gfmd/db_ldap.c 2011-05-26 06:37:17 UTC (rev 5354) +++ gfarm_v2/trunk/server/gfmd/db_ldap.c 2011-05-26 07:10:33 UTC (rev 5355) @@ -62,6 +62,7 @@ #include "config.h" #include "metadb_common.h" #include "xattr_info.h" +#include "metadb_server.h" #include "subr.h" #include "db_access.h" @@ -3180,4 +3181,9 @@ NULL, NULL, NULL, + + NULL, + NULL, + NULL, + NULL, }; Modified: gfarm_v2/trunk/server/gfmd/db_none.c =================================================================== --- gfarm_v2/trunk/server/gfmd/db_none.c 2011-05-26 06:37:17 UTC (rev 5354) +++ gfarm_v2/trunk/server/gfmd/db_none.c 2011-05-26 07:10:33 UTC (rev 5355) @@ -27,6 +27,7 @@ #include "config.h" #include "quota.h" +#include "metadb_server.h" #include "db_access.h" #include "db_ops.h" @@ -399,6 +400,34 @@ /**********************************************************************/ +static gfarm_error_t +gfarm_none_mdhost_add(gfarm_uint64_t seqnum, struct gfarm_metadb_server *ms) +{ + return (GFARM_ERR_OPERATION_NOT_SUPPORTED); +} + +static gfarm_error_t +gfarm_none_mdhost_modify(gfarm_uint64_t seqnum, + struct db_mdhost_modify_arg *arg) +{ + return (GFARM_ERR_OPERATION_NOT_SUPPORTED); +} + +static gfarm_error_t +gfarm_none_mdhost_remove(gfarm_uint64_t seqnum, char *name) +{ + return (GFARM_ERR_OPERATION_NOT_SUPPORTED); +} + +static gfarm_error_t +gfarm_none_mdhost_load(void *closure, + void (*callback)(void *, struct gfarm_metadb_server *)) +{ + return (GFARM_ERR_OPERATION_NOT_SUPPORTED); +} + +/**********************************************************************/ + const struct db_ops db_none_ops = { gfarm_none_initialize, gfarm_none_terminate, @@ -474,4 +503,9 @@ gfarm_none_seqnum_modify, gfarm_none_seqnum_remove, gfarm_none_seqnum_load, + + gfarm_none_mdhost_add, + gfarm_none_mdhost_modify, + gfarm_none_mdhost_remove, + gfarm_none_mdhost_load, }; Modified: gfarm_v2/trunk/server/gfmd/db_ops.h =================================================================== --- gfarm_v2/trunk/server/gfmd/db_ops.h 2011-05-26 06:37:17 UTC (rev 5354) +++ gfarm_v2/trunk/server/gfmd/db_ops.h 2011-05-26 07:10:33 UTC (rev 5355) @@ -129,6 +129,11 @@ struct gfarm_quota_info; +struct db_mdhost_modify_arg { + struct gfarm_metadb_server ms; + int modflags; +}; + struct db_ops { gfarm_error_t (*initialize)(void); gfarm_error_t (*terminate)(void); @@ -241,4 +246,12 @@ gfarm_error_t (*seqnum_remove)(char *); gfarm_error_t (*seqnum_load)(void *, void (*)(void *, struct db_seqnum_arg *)); + + gfarm_error_t (*mdhost_add)(gfarm_uint64_t, + struct gfarm_metadb_server *); + gfarm_error_t (*mdhost_modify)(gfarm_uint64_t, + struct db_mdhost_modify_arg *); + gfarm_error_t (*mdhost_remove)(gfarm_uint64_t, char *); + gfarm_error_t (*mdhost_load)(void *, + void (*)(void *, struct gfarm_metadb_server *)); }; Modified: gfarm_v2/trunk/server/gfmd/db_pgsql.c =================================================================== --- gfarm_v2/trunk/server/gfmd/db_pgsql.c 2011-05-26 06:37:17 UTC (rev 5354) +++ gfarm_v2/trunk/server/gfmd/db_pgsql.c 2011-05-26 07:10:33 UTC (rev 5355) @@ -32,6 +32,7 @@ #include "metadb_common.h" #include "xattr_info.h" #include "quota_info.h" +#include "metadb_server.h" #include "quota.h" #include "db_common.h" @@ -3274,6 +3275,140 @@ return (GFARM_ERR_NO_ERROR); } +/**********************************************************************/ + +static gfarm_error_t +mdhost_set_field(PGresult *res, int startrow, void *vinfo) +{ + struct gfarm_metadb_server *info = vinfo; + + info->name = pgsql_get_string_ck(res, startrow, "hostname"); + info->port = pgsql_get_int32(res, startrow, "port"); + info->clustername = pgsql_get_string_ck(res, startrow, "clustername"); + info->flags = pgsql_get_int32(res, startrow, "flags"); + return (GFARM_ERR_NO_ERROR); +} + +static gfarm_error_t +pgsql_mdhost_update(gfarm_uint64_t seqnum, struct gfarm_metadb_server *info, + const char *sql, + gfarm_error_t (*check)(PGresult *, const char *, const char *), + const char *diag) +{ + PGresult *res; + const char *paramValues[4]; + gfarm_error_t e; + char port[GFARM_INT32STRLEN + 1]; + char flags[GFARM_INT32STRLEN + 1]; + + if ((e = gfarm_pgsql_start(diag)) != GFARM_ERR_NO_ERROR) { + gflog_debug(GFARM_MSG_UNFIXED, + "gfarm_pgsql_start() failed"); + return (e); + } + paramValues[0] = info->name; + sprintf(port, "%d", info->port); + paramValues[1] = port; + paramValues[2] = info->clustername; + sprintf(flags, "%d", info->flags); + paramValues[3] = flags; + res = PQexecParams(conn, + sql, + 4, /* number of params */ + NULL, /* param types */ + paramValues, + NULL, /* param lengths */ + NULL, /* param formats */ + 0); /* ask for text results */ + if ((e = (*check)(res, sql, diag)) + == GFARM_ERR_DB_ACCESS_SHOULD_BE_RETRIED) + return (e); + PQclear(res); + + if (e == GFARM_ERR_NO_ERROR) + e = gfarm_pgsql_commit_sn(seqnum, diag); + else + gfarm_pgsql_rollback(diag); + return (e); +} + +static gfarm_error_t +gfarm_pgsql_mdhost_add(gfarm_uint64_t seqnum, struct gfarm_metadb_server *info) +{ + gfarm_error_t e; + + e = pgsql_mdhost_update(seqnum, info, + "INSERT INTO MDHost (hostname, port, clustername, flags) " + "VALUES ($1, $2, $3, $4)", + gfarm_pgsql_check_insert, "pgsql_mdhost_add"); + + FREE_ARG(info); + return (e); +} + +static gfarm_error_t +gfarm_pgsql_mdhost_modify(gfarm_uint64_t seqnum, + struct db_mdhost_modify_arg *arg) +{ + gfarm_error_t e; + + e = pgsql_mdhost_update(seqnum, &arg->ms, + "UPDATE MDHost " + "SET port = $2, clustername = $3, flags = $4 " + "WHERE hostname = $1", + gfarm_pgsql_check_update_or_delete, "pgsql_mdhost_modify"); + + FREE_ARG(arg); + return (e); +} + +static gfarm_error_t +gfarm_pgsql_mdhost_remove(gfarm_uint64_t seqnum, char *hostname) +{ + gfarm_error_t e; + const char *paramValues[1]; + + paramValues[0] = hostname; + e = gfarm_pgsql_update_or_delete(seqnum, + "DELETE FROM MDHost WHERE hostname = $1", + 1, /* number of params */ + NULL, /* param types */ + paramValues, + NULL, /* param lengths */ + NULL, /* param formats */ + 0, /* ask for text results */ + "pgsql_mdhost_remove"); + + FREE_ARG(hostname); + return (e); +} + +static gfarm_error_t +gfarm_pgsql_mdhost_load(void *closure, + void (*callback)(void *, struct gfarm_metadb_server *)) +{ + gfarm_error_t e; + int i, n; + struct gfarm_metadb_server *infos; + + e = gfarm_pgsql_generic_get_all_no_retry( + "SELECT * FROM MDHost", + 0, NULL, + &n, &infos, + &gfarm_base_metadb_server_ops, mdhost_set_field, + "pgsql_mdhost_load"); + if (e != GFARM_ERR_NO_ERROR) { + gflog_debug(GFARM_MSG_UNFIXED, + "gfarm_pgsql_generic_get_all_no_retry()"); + return (e); + } + for (i = 0; i < n; i++) + (*callback)(closure, &infos[i]); + + free(infos); + return (GFARM_ERR_NO_ERROR); +} + #endif /**********************************************************************/ @@ -3363,10 +3498,20 @@ gfarm_pgsql_seqnum_modify, gfarm_pgsql_seqnum_remove, gfarm_pgsql_seqnum_load, + + gfarm_pgsql_mdhost_add, + gfarm_pgsql_mdhost_modify, + gfarm_pgsql_mdhost_remove, + gfarm_pgsql_mdhost_load, #else NULL, NULL, NULL, NULL, + + NULL, + NULL, + NULL, + NULL, #endif }; Modified: gfarm_v2/trunk/server/gfmd/gfmd.c =================================================================== --- gfarm_v2/trunk/server/gfmd/gfmd.c 2011-05-26 06:37:17 UTC (rev 5354) +++ gfarm_v2/trunk/server/gfmd/gfmd.c 2011-05-26 07:10:33 UTC (rev 5355) @@ -56,6 +56,7 @@ #include "db_journal_apply.h" #include "host.h" #include "mdhost.h" +#include "mdcluster.h" #include "user.h" #include "group.h" #include "peer.h" @@ -510,6 +511,21 @@ case GFM_PROTO_QUOTA_CHECK: e = gfm_server_quota_check(peer, from_client, skip); break; + case GFM_PROTO_METADB_SERVER_GET: + e = gfm_server_metadb_server_get(peer, from_client, skip); + break; + case GFM_PROTO_METADB_SERVER_GET_ALL: + e = gfm_server_metadb_server_get_all(peer, from_client, skip); + break; + case GFM_PROTO_METADB_SERVER_SET: + e = gfm_server_metadb_server_set(peer, from_client, skip); + break; + case GFM_PROTO_METADB_SERVER_MODIFY: + e = gfm_server_metadb_server_modify(peer, from_client, skip); + break; + case GFM_PROTO_METADB_SERVER_REMOVE: + e = gfm_server_metadb_server_remove(peer, from_client, skip); + break; default: e = gfm_server_protocol_extension(peer, from_client, skip, level, request, requestp, on_errorp); @@ -1033,6 +1049,13 @@ "because this is already the master gfmd"); return; } + if (!mdhost_self_is_master_candidate()) { + gflog_error(GFARM_MSG_UNFIXED, + "cannot transform to the master gfmd " + "because this is not a master candidate."); + return; + } + master = mdhost_lookup_master(); if (mdhost_is_up(master)) mdhost_disconnect(master, NULL); @@ -1074,6 +1097,31 @@ gfarm_mutex_unlock(&transform_mutex, diag, TRANSFORM_MUTEX_DIAG); return (open_accepting_socket(gfarm_metadb_server_port)); } + +static int +mdcluster_foreach_select_master(struct mdhost *mh, void *closure) +{ + struct mdhost **mhp = closure; + + if ((*mhp) != mh) { + *mhp = mh; + return (0); + } + return (1); +} + +static void +select_master(void) +{ + struct mdhost *self = mdhost_lookup_self(), *mh = self; + + mdcluster_foreach_mdhost(mdhost_get_cluster(self), + mdcluster_foreach_select_master, &mh); + if (self != mh) { + mdhost_set_is_master(self, 0); + mdhost_set_is_master(mh, 1); + } +} #endif static void @@ -1247,12 +1295,15 @@ callout_module_init(CALLOUT_NTHREADS); - back_channel_init(); #ifdef ENABLE_METADATA_REPLICATION - gfmdc_init(); /* must be called before db_journal_init() */ db_journal_apply_init(); - db_journal_init(mdhost_self_is_master()); + db_journal_init(); #endif + mdhost_init(); + back_channel_init(); +#ifdef ENABLE_METADATA_REPLICATION + gfmdc_init(); +#endif /* directory service */ host_init(); user_init(); @@ -1286,6 +1337,9 @@ int syslog_facility = GFARM_DEFAULT_FACILITY; int ch, sock, table_size; sigset_t sigs; +#ifdef ENABLE_METADATA_REPLICATION + int is_master; +#endif if (argc >= 1) program_name = basename(argv[0]); @@ -1354,14 +1408,6 @@ if (syslog_level != -1) gflog_set_priority_level(syslog_level); - if (port_number != NULL) - gfarm_metadb_server_port = strtol(port_number, NULL, 0); - mdhost_init(); - if (mdhost_self_is_master()) - sock = open_accepting_socket(gfarm_metadb_server_port); - else - sock = -1; - /* * We do this before calling gfarm_daemon() * to print the error message to stderr. @@ -1468,13 +1514,22 @@ inode_check_and_repair() */ inode_check_and_repair(); } + if (port_number != NULL) + gfarm_metadb_server_port = strtol(port_number, NULL, 0); #ifdef ENABLE_METADATA_REPLICATION + if (mdhost_self_is_master() && gfarm_get_metadb_server_force_slave()) + select_master(); + is_master = mdhost_self_is_master(); gflog_info(GFARM_MSG_UNFIXED, "metadata replication %s mode", - mdhost_self_is_master() ? "master" : "slave"); + is_master ? "master" : "slave"); start_gfmdc_threads(); - if (sock == -1) + if (is_master) + sock = open_accepting_socket(gfarm_metadb_server_port); + else sock = wait_transform_to_master(); +#else + sock = open_accepting_socket(gfarm_metadb_server_port); #endif accepting_loop(sock); Modified: gfarm_v2/trunk/server/gfmd/gfmd_channel.c =================================================================== --- gfarm_v2/trunk/server/gfmd/gfmd_channel.c 2011-05-26 06:37:17 UTC (rev 5354) +++ gfarm_v2/trunk/server/gfmd/gfmd_channel.c 2011-05-26 07:10:33 UTC (rev 5355) @@ -26,8 +26,6 @@ #include "auth.h" #include "gfm_client.h" #include "config.h" -#include "metadb_server.h" -#include "filesystem.h" #include "peer.h" #include "subr.h" @@ -70,7 +68,7 @@ static struct thread_pool *journal_sync_thread_pool; static struct gfmdc_journal_sync_info journal_sync_info; -#define BACK_CHANNEL_DIAG "gfmd_channel" +#define CHANNEL_DIAG "gfmd_channel" #define SYNC_MUTEX_DIAG "jorunal_sync_info.sync_mutex" #define SYNC_END_COND_DIAG "jorunal_sync_info.sync_end_cond" #define SEND_MUTEX_DIAG "send_closure.mutex" @@ -94,7 +92,7 @@ } static gfarm_error_t -gfmdc_server_put_reply(struct mdhost *host, +gfmdc_server_put_reply(struct mdhost *mh, struct peer *peer, gfp_xdr_xid_t xid, const char *diag, gfarm_error_t errcode, char *format, ...) { @@ -103,15 +101,15 @@ va_start(ap, format); e = gfm_server_channel_vput_reply( - mdhost_to_abstract_host(host), peer, xid, diag, - errcode, format, &ap, BACK_CHANNEL_DIAG); + mdhost_to_abstract_host(mh), peer, xid, diag, + errcode, format, &ap); va_end(ap); return (e); } static gfarm_error_t -gfmdc_client_recv_result(struct peer *peer, struct mdhost *host, +gfmdc_client_recv_result(struct peer *peer, struct mdhost *mh, size_t size, const char *diag, const char *format, ...) { gfarm_error_t e, errcode; @@ -119,7 +117,7 @@ va_start(ap, format); e = gfm_client_channel_vrecv_result(peer, - mdhost_to_abstract_host(host), size, diag, + mdhost_to_abstract_host(mh), size, diag, &format, &errcode, &ap); va_end(ap); if (e != GFARM_ERR_NO_ERROR) @@ -133,7 +131,7 @@ } static gfarm_error_t -gfmdc_client_send_request(struct mdhost *host, +gfmdc_client_send_request(struct mdhost *mh, struct peer *peer0, const char *diag, gfarm_int32_t (*result_callback)(void *, void *, size_t), void (*disconnect_callback)(void *, void *), @@ -146,12 +144,12 @@ va_start(ap, format); /* XXX FIXME gfm_client_channel_vsend_request must be async-request */ e = gfm_client_channel_vsend_request( - mdhost_to_abstract_host(host), peer0, diag, + mdhost_to_abstract_host(mh), peer0, diag, result_callback, disconnect_callback, closure, #ifdef COMPAT_GFARM_2_3 NULL, #endif - command, format, &ap, BACK_CHANNEL_DIAG); + command, format, &ap); va_end(ap); return (e); } @@ -170,9 +168,9 @@ static void gfmdc_journal_send_closure_reset(struct gfmdc_journal_send_closure *c, - struct mdhost *host) + struct mdhost *mh) { - c->host = host; + c->host = mh; c->data = NULL; } @@ -267,32 +265,32 @@ char *data; gfarm_uint64_t min_seqnum, from_sn, to_sn, lf_sn; struct journal_file_reader *reader; - struct mdhost *host = c->host; + struct mdhost *mh = c->host; static const char *diag = "GFM_PROTO_JOURNAL_SEND"; - lf_sn = mdhost_get_last_fetch_seqnum(host); + lf_sn = mdhost_get_last_fetch_seqnum(mh); min_seqnum = lf_sn == 0 ? 0 : lf_sn + 1; - reader = mdhost_get_journal_file_reader(host); + reader = mdhost_get_journal_file_reader(mh); assert(reader); e = db_journal_fetch(reader, min_seqnum, &data, &data_len, &from_sn, - &to_sn, &no_rec, mdhost_get_name(host)); + &to_sn, &no_rec, mdhost_get_name(mh)); if (e != GFARM_ERR_NO_ERROR) { gflog_error(GFARM_MSG_UNFIXED, "%s : %s", - mdhost_get_name(host), gfarm_error_string(e)); + mdhost_get_name(mh), gfarm_error_string(e)); return (e); } else if (no_rec) { *to_snp = 0; return (GFARM_ERR_NO_ERROR); } - mdhost_set_last_fetch_seqnum(host, to_sn); + mdhost_set_last_fetch_seqnum(mh, to_sn); c->data = data; - if ((e = gfmdc_client_send_request(host, NULL, diag, + if ((e = gfmdc_client_send_request(mh, NULL, diag, result_op, disconnect_op, c, GFM_PROTO_JOURNAL_SEND, "llb", from_sn, to_sn, (size_t)data_len, data)) != GFARM_ERR_NO_ERROR) { gflog_error(GFARM_MSG_UNFIXED, - "%s : %s", mdhost_get_name(host), gfarm_error_string(e)); + "%s : %s", mdhost_get_name(mh), gfarm_error_string(e)); free(data); c->data = NULL; return (e); @@ -329,7 +327,7 @@ } static gfarm_error_t -gfmdc_server_journal_send(struct mdhost *host, struct peer *peer, +gfmdc_server_journal_send(struct mdhost *mh, struct peer *peer, gfp_xdr_xid_t xid, size_t size) { gfarm_error_t e, er; @@ -346,17 +344,17 @@ if (er == GFARM_ERR_NO_ERROR) gflog_debug(GFARM_MSG_UNFIXED, "from %s : recv journal %llu to %llu", - mdhost_get_name(host), (unsigned long long)from_sn, + mdhost_get_name(mh), (unsigned long long)from_sn, (unsigned long long)to_sn); #endif - e = gfmdc_server_put_reply(host, peer, xid, diag, er, ""); + e = gfmdc_server_put_reply(mh, peer, xid, diag, er, ""); return (e); } static void* gfmdc_journal_first_sync_thread(void *); static gfarm_error_t -gfmdc_server_journal_ready_to_recv(struct mdhost *host, struct peer *peer, +gfmdc_server_journal_ready_to_recv(struct mdhost *mh, struct peer *peer, gfp_xdr_xid_t xid, size_t size) { gfarm_error_t e; @@ -366,32 +364,33 @@ if ((e = gfmdc_server_get_request(peer, size, diag, "l", &seqnum)) == GFARM_ERR_NO_ERROR) { - mdhost_set_last_fetch_seqnum(host, seqnum); - mdhost_set_is_recieved_seqnum(host, 1); + mdhost_set_last_fetch_seqnum(mh, seqnum); + mdhost_set_is_recieved_seqnum(mh, 1); #ifdef DEBUG_JOURNAL gflog_debug(GFARM_MSG_UNFIXED, "%s : last_fetch_seqnum=%llu", - mdhost_get_name(host), (unsigned long long)seqnum); + mdhost_get_name(mh), (unsigned long long)seqnum); #endif - reader = mdhost_get_journal_file_reader(host); + reader = mdhost_get_journal_file_reader(mh); if (reader == NULL || journal_file_reader_is_invalid(reader)) { if ((e = db_journal_reader_reopen(&reader, - mdhost_get_last_fetch_seqnum(host))) + mdhost_get_last_fetch_seqnum(mh))) != GFARM_ERR_NO_ERROR) { gflog_error(GFARM_MSG_UNFIXED, "gfmd_channel(%s) : %s", - mdhost_get_name(host), + mdhost_get_name(mh), gfarm_error_string(e)); } else - mdhost_set_journal_file_reader(host, reader); + mdhost_set_journal_file_reader(mh, reader); } } - e = gfmdc_server_put_reply(host, peer, xid, diag, e, ""); - if (mdhost_is_sync_replication(host)) { + mdhost_activate(mh); + e = gfmdc_server_put_reply(mh, peer, xid, diag, e, ""); + if (mdhost_is_sync_replication(mh)) { gfarm_mutex_lock(&journal_sync_info.sync_mutex, diag, SYNC_MUTEX_DIAG); thrpool_add_job(journal_sync_thread_pool, - gfmdc_journal_first_sync_thread, host); + gfmdc_journal_first_sync_thread, mh); gfarm_mutex_unlock(&journal_sync_info.sync_mutex, diag, SYNC_MUTEX_DIAG); } @@ -403,13 +402,13 @@ { gfarm_error_t e; struct peer *peer = p; - struct mdhost *host = peer_get_mdhost(peer); + struct mdhost *mh = peer_get_mdhost(peer); static const char *diag = "GFM_PROTO_JOURNAL_READY_TO_RECV"; - if ((e = gfmdc_client_recv_result(peer, host, size, diag, "")) + if ((e = gfmdc_client_recv_result(peer, mh, size, diag, "")) != GFARM_ERR_NO_ERROR) gflog_error(GFARM_MSG_UNFIXED, "%s : %s", - mdhost_get_name(host), gfarm_error_string(e)); + mdhost_get_name(mh), gfarm_error_string(e)); return (e); } @@ -419,7 +418,7 @@ } gfarm_error_t -gfmdc_client_journal_ready_to_recv(struct mdhost *host) +gfmdc_client_journal_ready_to_recv(struct mdhost *mh) { gfarm_error_t e; gfarm_uint64_t seqnum; @@ -429,13 +428,13 @@ seqnum = db_journal_get_current_seqnum(); giant_unlock(); - if ((e = gfmdc_client_send_request(host, NULL, diag, + if ((e = gfmdc_client_send_request(mh, NULL, diag, gfmdc_client_journal_ready_to_recv_result, gfmdc_client_journal_ready_to_recv_disconnect, NULL, GFM_PROTO_JOURNAL_READY_TO_RECV, "l", seqnum)) != GFARM_ERR_NO_ERROR) gflog_error(GFARM_MSG_UNFIXED, - "%s : %s", m... [truncated message content] |