From: <ta...@us...> - 2007-06-08 17:30:43
|
Revision: 3751 http://gfarm.svn.sourceforge.net/gfarm/?rev=3751&view=rev Author: tatebe Date: 2007-06-08 10:30:36 -0700 (Fri, 08 Jun 2007) Log Message: ----------- support replica creation by gfs_replicate_from_to or gfrep command. Modified Paths: -------------- gfarm/branches/gfarm_v2/gftool/Makefile gfarm/branches/gfarm_v2/gftool/gfrep/Makefile gfarm/branches/gfarm_v2/include/gfarm/gfs.h gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/config.h gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/config_client.c gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/gfs_client.c gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/gfs_client.h gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/gfs_pio.c gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/gfs_proto.h gfarm/branches/gfarm_v2/server/gfmd/fs.c gfarm/branches/gfarm_v2/server/gfmd/inode.c gfarm/branches/gfarm_v2/server/gfmd/inode.h gfarm/branches/gfarm_v2/server/gfmd/process.c gfarm/branches/gfarm_v2/server/gfmd/process.h gfarm/branches/gfarm_v2/server/gfsd/gfsd.c Added Paths: ----------- gfarm/branches/gfarm_v2/gftool/gfrep/gfrep_simple.c gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/gfs_replicate.c Modified: gfarm/branches/gfarm_v2/gftool/Makefile =================================================================== --- gfarm/branches/gfarm_v2/gftool/Makefile 2007-06-07 01:06:48 UTC (rev 3750) +++ gfarm/branches/gfarm_v2/gftool/Makefile 2007-06-08 17:30:36 UTC (rev 3751) @@ -14,6 +14,7 @@ gfmkdir \ gfmv \ gfreg \ + gfrep \ gfrm \ gfrmdir \ gfstat \ @@ -26,7 +27,6 @@ # gfifo \ # gfpwd \ # gfrcmd \ -# gfrep \ # gfsched \ # gfsck \ # gfsetdir \ Modified: gfarm/branches/gfarm_v2/gftool/gfrep/Makefile =================================================================== --- gfarm/branches/gfarm_v2/gftool/gfrep/Makefile 2007-06-07 01:06:48 UTC (rev 3750) +++ gfarm/branches/gfarm_v2/gftool/gfrep/Makefile 2007-06-08 17:30:36 UTC (rev 3751) @@ -6,8 +6,8 @@ include $(top_srcdir)/makes/var.mk -SRCS = gfrep.c -OBJS = gfrep.o +SRCS = gfrep_simple.c +OBJS = gfrep_simple.o PROGRAM = gfrep #CC = omcc #OMPFLAGS = -fopenmp @@ -21,4 +21,4 @@ ### -$(OBJS): $(DEPGFARMINC) $(GFARMLIB_SRCDIR)/host.h $(GFARMLIB_SRCDIR)/schedule.h $(GFARMLIB_SRCDIR)/gfs_client.h $(GFARMLIB_SRCDIR)/gfs_misc.h $(GFARMLIB_SRCDIR)/gfarm_list.h $(GFARMLIB_SRCDIR)/gfarm_foreach.h $(GFARMLIB_SRCDIR)/gfarm_xinfo.h $(GFUTIL_SRCDIR)/hash.h +$(OBJS): $(DEPGFARMINC) $(GFARMLIB_SRCDIR)/metadb_server.h Added: gfarm/branches/gfarm_v2/gftool/gfrep/gfrep_simple.c =================================================================== --- gfarm/branches/gfarm_v2/gftool/gfrep/gfrep_simple.c (rev 0) +++ gfarm/branches/gfarm_v2/gftool/gfrep/gfrep_simple.c 2007-06-08 17:30:36 UTC (rev 3751) @@ -0,0 +1,87 @@ +/* + * $Id$ + */ + +#include <unistd.h> +#include <stdlib.h> +#include <libgen.h> +#include <stdio.h> + +#include <gfarm/gfarm.h> + +#include "metadb_server.h" + +char *program_name = "gfrep"; + +static int +usage() +{ + fprintf(stderr, "Usage: %s -s srchost -d dsthost file\n", + program_name); + exit(EXIT_FAILURE); +} + +int +main(int argc, char *argv[]) +{ + char *src = NULL, *dst = NULL, *f, c; + struct gfarm_host_info sinfo, dinfo; + gfarm_error_t e; + + if (argc > 0) + program_name = basename(argv[0]); + + e = gfarm_initialize(&argc, &argv); + if (e != GFARM_ERR_NO_ERROR) { + fprintf(stderr, "%s: %s\n", + program_name, gfarm_error_string(e)); + exit(EXIT_FAILURE); + } + while ((c = getopt(argc, argv, "d:s:h?")) != -1) { + switch (c) { + case 'd': + dst = optarg; + break; + case 's': + src = optarg; + break; + case 'h': + case '?': + default: + usage(); + } + } + argc -= optind; + argv += optind; + + if (src == NULL || dst == NULL) + usage(); + + e = gfarm_host_info_get_by_name_alias(src, &sinfo); + if (e != GFARM_ERR_NO_ERROR) { + fprintf(stderr, "%s: %s\n", src, gfarm_error_string(e)); + exit(EXIT_FAILURE); + } + e = gfarm_host_info_get_by_name_alias(dst, &dinfo); + if (e != GFARM_ERR_NO_ERROR) { + fprintf(stderr, "%s: %s\n", dst, gfarm_error_string(e)); + exit(EXIT_FAILURE); + } + + f = *argv; + e = gfs_replicate_from_to(f, src, sinfo.port, dst, dinfo.port); + if (e != GFARM_ERR_NO_ERROR) { + fprintf(stderr, "%s: %s\n", f, gfarm_error_string(e)); + exit(EXIT_FAILURE); + } + gfarm_host_info_free(&sinfo); + gfarm_host_info_free(&dinfo); + + e = gfarm_terminate(); + if (e != GFARM_ERR_NO_ERROR) { + fprintf(stderr, "%s: %s\n", + program_name, gfarm_error_string(e)); + exit(EXIT_FAILURE); + } + return (0); +} Modified: gfarm/branches/gfarm_v2/include/gfarm/gfs.h =================================================================== --- gfarm/branches/gfarm_v2/include/gfarm/gfs.h 2007-06-07 01:06:48 UTC (rev 3750) +++ gfarm/branches/gfarm_v2/include/gfarm/gfs.h 2007-06-08 17:30:36 UTC (rev 3751) @@ -227,6 +227,8 @@ #define GFS_W_OK 2 #define GFS_R_OK 4 +gfarm_error_t gfs_replicate_from_to(char *, char *, int, char *, int); + gfarm_error_t gfs_execve(const char *, char *const *, char *const *); /* XXX need gfs_statfsnode() equivalent */ Modified: gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/config.h =================================================================== --- gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/config.h 2007-06-07 01:06:48 UTC (rev 3750) +++ gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/config.h 2007-06-08 17:30:36 UTC (rev 3751) @@ -80,3 +80,5 @@ /* for client */ struct gfs_connection; gfarm_error_t gfarm_client_process_set(struct gfs_connection *); +gfarm_error_t gfarm_client_replica_add_from(struct gfs_connection *, + char *, gfarm_int32_t, gfarm_int32_t); Modified: gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/config_client.c =================================================================== --- gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/config_client.c 2007-06-07 01:06:48 UTC (rev 3750) +++ gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/config_client.c 2007-06-08 17:30:36 UTC (rev 3751) @@ -432,6 +432,17 @@ gfarm_client_pid)); } +gfarm_error_t +gfarm_client_replica_add_from(struct gfs_connection *gfs_server, + char *host, gfarm_int32_t port, gfarm_int32_t fd) +{ + return (gfs_client_replica_add_from(gfs_server, + gfarm_client_pid_key_type, + gfarm_client_pid_key_len, gfarm_client_pid_key, + gfarm_client_pid, + host, port, fd)); +} + /* * the following function is for client, * server/daemon process shouldn't call it. Modified: gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/gfs_client.c =================================================================== --- gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/gfs_client.c 2007-06-07 01:06:48 UTC (rev 3750) +++ gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/gfs_client.c 2007-06-08 17:30:36 UTC (rev 3751) @@ -1200,12 +1200,14 @@ } gfarm_error_t -gfs_client_replica_add(struct gfs_connection *gfs_server, gfarm_int32_t fd) +gfs_client_replica_add_from(struct gfs_connection *gfs_server, + gfarm_int32_t type, size_t length, const char *key, gfarm_pid_t pid, + char *host, gfarm_int32_t port, gfarm_int32_t fd) { gfs_client_connection_used(gfs_server); - return (gfs_client_rpc(gfs_server, 0, GFS_PROTO_REPLICA_ADD, "i/", - fd)); + return (gfs_client_rpc(gfs_server, 0, GFS_PROTO_REPLICA_ADD_FROM, + "iblsii/", type, length, key, pid, host, port, fd)); } gfarm_error_t @@ -1369,9 +1371,81 @@ /* * GFS_PROTO_REPLICA_RECV is only used by gfsd, - * thus, we define the client protocol at gfsd instead of here. + * but defined here for better maintainability. */ +gfarm_error_t +gfs_client_replica_recv(struct gfs_connection *gfs_server, + gfarm_int32_t net_fd, gfarm_int32_t local_fd) +{ + gfarm_error_t e, e_write = GFARM_ERR_NO_ERROR, e_rpc; + int i, rv, eof; + char buffer[GFS_PROTO_MAX_IOSIZE]; + + gfs_client_connection_used(gfs_server); + + e = gfs_client_rpc_request(gfs_server, GFS_PROTO_REPLICA_RECV, "i", + net_fd); + if (e != GFARM_ERR_NO_ERROR) + return (e); + + for (;;) { + gfarm_int32_t size; + int skip = 0; + + /* XXX - FIXME layering violation */ + e = gfp_xdr_recv(gfs_server->conn, 0, &eof, "i", &size); + if (e != GFARM_ERR_NO_ERROR) + break; + if (eof) { + e = GFARM_ERR_PROTOCOL; + break; + } + if (size <= 0) + break; + do { + /* XXX - FIXME layering violation */ + int partial = gfp_xdr_recv_partial(gfs_server->conn, 0, + buffer, size); + + if (partial <= 0) + return (GFARM_ERR_PROTOCOL); + size -= partial; +#ifdef __GNUC__ /* shut up stupid warning by gcc */ + rv = 0; +#endif + i = 0; + if (skip) /* write(2) returns error */ + i = partial; + for (; i < partial; i += rv) { + rv = write(local_fd, buffer + i, partial - i); + if (rv <= 0) + break; + } + if (i < partial) { + /* + * write(2) never returns 0, + * so the following rv == 0 case is + * just warm fuzzy. + */ + e_write = gfarm_errno_to_error( + rv == 0 ? ENOSPC : errno); + /* + * we should receive rest of data, + * even if write(2) fails. + */ + skip = 1; + } + } while (size > 0); + } + e_rpc = gfs_client_rpc_result(gfs_server, 0, ""); + if (e == GFARM_ERR_NO_ERROR) + e = e_write; + if (e == GFARM_ERR_NO_ERROR) + e = e_rpc; + return (e); +} + /* ********************************************************************** * Implementation of gfs_client_command() Modified: gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/gfs_client.h =================================================================== --- gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/gfs_client.h 2007-06-07 01:06:48 UTC (rev 3750) +++ gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/gfs_client.h 2007-06-08 17:30:36 UTC (rev 3751) @@ -70,7 +70,11 @@ gfarm_error_t gfs_client_lock_info(struct gfs_connection *, gfarm_int32_t, gfarm_off_t, gfarm_off_t, gfarm_int32_t, gfarm_int32_t, gfarm_off_t *, gfarm_off_t *, gfarm_int32_t *, char**, gfarm_pid_t **); -gfarm_error_t gfs_client_replica_add(struct gfs_connection *, gfarm_int32_t); +gfarm_error_t gfs_client_replica_add_from(struct gfs_connection *, + gfarm_int32_t, size_t, const char *, gfarm_pid_t, + char *, gfarm_int32_t, gfarm_int32_t); +gfarm_error_t gfs_client_replica_recv(struct gfs_connection *, + gfarm_int32_t, gfarm_int32_t); gfarm_error_t gfs_client_statfs(struct gfs_connection *, char *, gfarm_int32_t *, gfarm_off_t *, gfarm_off_t *, gfarm_off_t *, Modified: gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/gfs_pio.c =================================================================== --- gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/gfs_pio.c 2007-06-07 01:06:48 UTC (rev 3750) +++ gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/gfs_pio.c 2007-06-08 17:30:36 UTC (rev 3751) @@ -94,13 +94,10 @@ return (GFARM_ERR_NO_ERROR); } +/* gfs_pio_fileno returns a network-wide file descriptor in Gfarm v2 */ int gfs_pio_fileno(GFS_File gf) { - gfarm_error_t e = gfs_pio_check_view_default(gf); - if (e != GFARM_ERR_NO_ERROR) - return (-1); - - return ((*gf->ops->view_fd)(gf)); + return (gf == NULL ? -1 : gf->fd); } static gfarm_error_t @@ -258,11 +255,16 @@ GFARM_TIMEVAL_FIX_INITIALIZE_WARNING(t1); gfs_profile(gfarm_gettimerval(&t1)); - e_save = gfs_pio_check_view_default(gf); - if (e_save == GFARM_ERR_NO_ERROR) { - if ((gf->mode & GFS_FILE_MODE_WRITE) != 0) - e_save = gfs_pio_flush(gf); - + /* + * no need to check and set the default file view here + * because neither gfs_pio_flush nor view_close is not + * needed unless the file view is specified by some + * operation. + */ + e_save = GFARM_ERR_NO_ERROR; + if ((gf->mode & GFS_FILE_MODE_WRITE) != 0) + e_save = gfs_pio_flush(gf); + if (gf->ops != NULL) { e = (*gf->ops->view_close)(gf); if (e_save == GFARM_ERR_NO_ERROR) e_save = e; Modified: gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/gfs_proto.h =================================================================== --- gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/gfs_proto.h 2007-06-07 01:06:48 UTC (rev 3750) +++ gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/gfs_proto.h 2007-06-08 17:30:36 UTC (rev 3751) @@ -26,6 +26,7 @@ GFS_PROTO_LOCK_INFO, GFS_PROTO_REPLICA_ADD, + GFS_PROTO_REPLICA_ADD_FROM, GFS_PROTO_REPLICA_RECV, GFS_PROTO_STATFS, Added: gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/gfs_replicate.c =================================================================== --- gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/gfs_replicate.c (rev 0) +++ gfarm/branches/gfarm_v2/lib/libgfarm/gfarm/gfs_replicate.c 2007-06-08 17:30:36 UTC (rev 3751) @@ -0,0 +1,35 @@ +/* + * $Id$ + */ + +#include <sys/types.h> +#include <sys/socket.h> + +#include <gfarm/gfarm.h> + +#include "gfutil.h" +#include "config.h" +#include "gfs_client.h" + +gfarm_error_t +gfs_replicate_from_to(char *file, char *srchost, int srcport, + char *dsthost, int dstport) +{ + struct gfs_connection *server; + gfarm_error_t e, e2; + GFS_File gf; + + e = gfs_pio_open(file, GFARM_FILE_RDONLY, &gf); + if (e != GFARM_ERR_NO_ERROR) + return (e); + + e = gfs_client_connection_acquire_by_host(dsthost, dstport, &server); + if (e == GFARM_ERR_NO_ERROR) { + e = gfarm_client_replica_add_from( + server, srchost, srcport, gfs_pio_fileno(gf)); + gfs_client_connection_free(server); + } + + e2 = gfs_pio_close(gf); + return (e != GFARM_ERR_NO_ERROR ? e : e2); +} Modified: gfarm/branches/gfarm_v2/server/gfmd/fs.c =================================================================== --- gfarm/branches/gfarm_v2/server/gfmd/fs.c 2007-06-07 01:06:48 UTC (rev 3750) +++ gfarm/branches/gfarm_v2/server/gfmd/fs.c 2007-06-08 17:30:36 UTC (rev 3751) @@ -212,7 +212,7 @@ if (e != GFARM_ERR_NO_ERROR) return (e); if (created && !from_client) { - e = inode_add_replica(inode, spool_host); + e = inode_add_replica(inode, spool_host, 1); if (e != GFARM_ERR_NO_ERROR) { process_close_file(process, peer, fd); inode_unlink(base, name, process); @@ -1567,37 +1567,67 @@ gfm_server_replica_adding(struct peer *peer, int from_client, int skip) { gfarm_error_t e; + gfarm_ino_t inum; + gfarm_uint64_t gen; + gfarm_int64_t mtime_sec; + gfarm_int32_t fd, mtime_nsec; + struct host *spool_host; + struct process *process; - /* XXX - NOT IMPLEMENTED */ - gflog_error("replica_adding: not implemented"); + if (skip) + return (GFARM_ERR_NO_ERROR); + giant_lock(); - e = gfm_server_put_reply(peer, "replica_adding", - GFARM_ERR_FUNCTION_NOT_IMPLEMENTED, ""); - return (e != GFARM_ERR_NO_ERROR ? e : - GFARM_ERR_FUNCTION_NOT_IMPLEMENTED); + if (from_client) /* from gfsd only */ + e = GFARM_ERR_OPERATION_NOT_PERMITTED; + else if ((spool_host = peer_get_host(peer)) == NULL) + e = GFARM_ERR_OPERATION_NOT_PERMITTED; + else if ((process = peer_get_process(peer)) == NULL) + e = GFARM_ERR_OPERATION_NOT_PERMITTED; + else if ((e = peer_fdpair_get_current(peer, &fd)) != + GFARM_ERR_NO_ERROR) + ; + else + e = process_replica_adding(process, peer, spool_host, fd, + &inum, &gen, &mtime_sec, &mtime_nsec); + + giant_unlock(); + return (gfm_server_put_reply(peer, "replica_adding", e, "llli", + inum, gen, mtime_sec, mtime_nsec)); } gfarm_error_t gfm_server_replica_added(struct peer *peer, int from_client, int skip) { gfarm_error_t e; - gfarm_int32_t flags, mtime_nsec; + gfarm_int32_t fd, flags, mtime_nsec; gfarm_int64_t mtime_sec; + struct host *spool_host; + struct process *process; - /* XXX - NOT IMPLEMENTED */ - gflog_error("replica_added: not implemented"); - e = gfm_server_get_request(peer, "replica_added", "ili", &flags, &mtime_sec, &mtime_nsec); if (e != GFARM_ERR_NO_ERROR) return (e); if (skip) return (GFARM_ERR_NO_ERROR); + giant_lock(); - e = gfm_server_put_reply(peer, "replica_added", - GFARM_ERR_FUNCTION_NOT_IMPLEMENTED, ""); - return (e != GFARM_ERR_NO_ERROR ? e : - GFARM_ERR_FUNCTION_NOT_IMPLEMENTED); + if (from_client) /* from gfsd only */ + e = GFARM_ERR_OPERATION_NOT_PERMITTED; + else if ((spool_host = peer_get_host(peer)) == NULL) + e = GFARM_ERR_OPERATION_NOT_PERMITTED; + else if ((process = peer_get_process(peer)) == NULL) + e = GFARM_ERR_OPERATION_NOT_PERMITTED; + else if ((e = peer_fdpair_get_current(peer, &fd)) != + GFARM_ERR_NO_ERROR) + ; + else + e = process_replica_added(process, peer, spool_host, fd, + flags, mtime_sec, mtime_nsec); + + giant_unlock(); + return (gfm_server_put_reply(peer, "replica_added", e, "")); } gfarm_error_t Modified: gfarm/branches/gfarm_v2/server/gfmd/inode.c =================================================================== --- gfarm/branches/gfarm_v2/server/gfmd/inode.c 2007-06-07 01:06:48 UTC (rev 3750) +++ gfarm/branches/gfarm_v2/server/gfmd/inode.c 2007-06-08 17:30:36 UTC (rev 3751) @@ -37,6 +37,7 @@ struct file_copy { struct file_copy *host_next; struct host *host; + int valid; }; struct inode { @@ -359,7 +360,7 @@ { if (!inode_free_list_initialized) inode_free_list_init(); - + if (inode_free_list.u.l.next != &inode_free_list) return (inode_alloc_num(inode_free_list.u.l.next->i_number)); else @@ -395,6 +396,9 @@ inode->i_number, gfarm_error_string(e)); } +static gfarm_error_t +remove_replica_internal(struct inode *, struct file_copy *); + void inode_remove(struct inode *inode) { @@ -405,23 +409,9 @@ gfarm_error_t e; for (copy = inode->u.c.s.f.copies; copy != NULL; copy = cn) { - e = host_remove_replica(copy->host, - inode->i_number, inode->i_gen); - if (e != GFARM_ERR_NO_ERROR) - gflog_error("host_remove_replica: %s", - host_name(copy->host)); - else if ((e = db_deadfilecopy_add(inode->i_number, - inode->i_gen, host_name(copy->host))) != - GFARM_ERR_NO_ERROR) - gflog_error("db_deadfilecopy_add: %s", - gfarm_error_string(e)); + e = remove_replica_internal(inode, copy); cn = copy->host_next; free(copy); - if ((e = db_filecopy_remove( - inode->i_number, host_name(copy->host))) != - GFARM_ERR_NO_ERROR) - gflog_error("db_filecopy_remove: %s", - gfarm_error_string(e)); } inode_cksum_remove(inode); } else if (inode_is_dir(inode)) { @@ -559,7 +549,8 @@ for (copy = inode->u.c.s.f.copies; copy != NULL; copy = copy->host_next) { - n++; + if (copy->valid) + n++; } return (n); } @@ -1245,31 +1236,44 @@ } gfarm_error_t -inode_add_replica_internal(struct inode *inode, struct host *spool_host) +inode_add_replica_internal(struct inode *inode, struct host *spool_host, + int valid) { struct file_copy *copy; for (copy = inode->u.c.s.f.copies; copy != NULL; copy = copy->host_next) { - if (copy->host == spool_host) - return (GFARM_ERR_ALREADY_EXISTS); + if (copy->host == spool_host) { + if (copy->valid) + return (GFARM_ERR_ALREADY_EXISTS); + else { + copy->valid = valid; + return (GFARM_ERR_NO_ERROR); + } + } } GFARM_MALLOC(copy); if (copy == NULL) return (GFARM_ERR_NO_MEMORY); copy->host = spool_host; + copy->valid = valid; copy->host_next = inode->u.c.s.f.copies; inode->u.c.s.f.copies = copy; return (GFARM_ERR_NO_ERROR); } +/* + * 'valid == 0' means that the replica is not ready right now, and + * going to be created. + */ gfarm_error_t -inode_add_replica(struct inode *inode, struct host *spool_host) +inode_add_replica(struct inode *inode, struct host *spool_host, int valid) { - gfarm_error_t e = inode_add_replica_internal(inode, spool_host); + gfarm_error_t e = inode_add_replica_internal(inode, spool_host, valid); - if (e != GFARM_ERR_NO_ERROR) + if (e != GFARM_ERR_NO_ERROR || !valid) return (e); + e = db_filecopy_add(inode->i_number, host_name(spool_host)); if (e != GFARM_ERR_NO_ERROR) gflog_error("db_filecopy_add(%" GFARM_PRId64 ", %s): %s", @@ -1278,6 +1282,30 @@ return (GFARM_ERR_NO_ERROR); } +static gfarm_error_t +remove_replica_internal(struct inode *inode, struct file_copy *copy) +{ + gfarm_error_t e, e2 = GFARM_ERR_NO_ERROR; + + e = host_remove_replica(copy->host, inode->i_number, inode->i_gen); + if (e != GFARM_ERR_NO_ERROR) + gflog_error("host_remove_replica(%" GFARM_PRId64 ", %s): %s", + inode->i_number, host_name(copy->host), + gfarm_error_string(e)); + else if (copy->valid && (e = db_deadfilecopy_add(inode->i_number, + inode->i_gen, host_name(copy->host))) != GFARM_ERR_NO_ERROR) + gflog_error("db_deadfilecopy_add(%" GFARM_PRId64 ", %s): %s", + inode->i_number, host_name(copy->host), + gfarm_error_string(e)); + + if (copy->valid && (e2 = db_filecopy_remove(inode->i_number, + host_name(copy->host))) != GFARM_ERR_NO_ERROR) + gflog_error("db_filecopy_remove(%" GFARM_PRId64 ", %s): %s", + inode->i_number, host_name(copy->host), + gfarm_error_string(e2)); + return (e != GFARM_ERR_NO_ERROR ? e : e2); +} + gfarm_error_t inode_remove_replica(struct inode *inode, struct host *spool_host) { @@ -1288,18 +1316,14 @@ copyp = ©->host_next) { if (copy->host == spool_host) { *copyp = copy->host_next; - free(copy); - goto delete_db_filecopy; + break; } } - return (GFARM_ERR_NO_SUCH_OBJECT); + if (copy == NULL) + return (GFARM_ERR_NO_SUCH_OBJECT); -delete_db_filecopy: - e = db_filecopy_remove(inode->i_number, host_name(spool_host)); - if (e != GFARM_ERR_NO_ERROR) - gflog_error("db_filecopy_remove(%" GFARM_PRId64 ", %s): %s", - inode->i_number, host_name(spool_host), - gfarm_error_string(e)); + e = remove_replica_internal(inode, copy); + free(copy); return (GFARM_ERR_NO_ERROR); } @@ -1386,7 +1410,7 @@ for (copy = inode->u.c.s.f.copies; copy != NULL; copy = copy->host_next) { if (copy->host == spool_host) - return (1); + return (copy->valid); } return (0); } @@ -1504,7 +1528,7 @@ return (host_match); } /* not opened */ - if (inode->u.c.s.f.copies != NULL) + if (!inode_is_creating_file(inode)) return (inode_has_replica(inode, spool_host)); return (to_create); } @@ -1553,18 +1577,18 @@ } /* read access, or write access && no process is opening the file */ - if (inode->u.c.s.f.copies == NULL) + if (inode_is_creating_file(inode)) gflog_fatal("inode_schedule_file_reply: should be creating"); n = 0; for (copy = inode->u.c.s.f.copies; copy != NULL; copy = copy->host_next) { - if (host_is_up(copy->host)) + if (copy->valid && host_is_up(copy->host)) n++; } e_save = host_schedule_reply_n(peer, n, diag); for (copy = inode->u.c.s.f.copies; copy != NULL; copy = copy->host_next) { - if (host_is_up(copy->host)) { + if (copy->valid && host_is_up(copy->host)) { e = host_schedule_reply(copy->host, peer, diag); if (e_save == GFARM_ERR_NO_ERROR) e_save = e; @@ -1660,7 +1684,7 @@ gflog_error("file_copy_add_one: not file %" GFARM_PRId64,inum); } else if (host == NULL) { gflog_error("file_copy_add_one: no host %s", hostname); - } else if ((e = inode_add_replica_internal(inode, host)) != + } else if ((e = inode_add_replica_internal(inode, host, 1)) != GFARM_ERR_NO_ERROR){ gflog_error("file_copy_add_one: add_replica: %s", gfarm_error_string(e)); Modified: gfarm/branches/gfarm_v2/server/gfmd/inode.h =================================================================== --- gfarm/branches/gfarm_v2/server/gfmd/inode.h 2007-06-07 01:06:48 UTC (rev 3750) +++ gfarm/branches/gfarm_v2/server/gfmd/inode.h 2007-06-08 17:30:36 UTC (rev 3751) @@ -54,7 +54,7 @@ struct process *); gfarm_error_t inode_unlink(struct inode *, char *, struct process *); -gfarm_error_t inode_add_replica(struct inode *, struct host *); +gfarm_error_t inode_add_replica(struct inode *, struct host *, int); gfarm_error_t inode_remove_replica(struct inode *, struct host *); struct file_opening; Modified: gfarm/branches/gfarm_v2/server/gfmd/process.c =================================================================== --- gfarm/branches/gfarm_v2/server/gfmd/process.c 2007-06-07 01:06:48 UTC (rev 3750) +++ gfarm/branches/gfarm_v2/server/gfmd/process.c 2007-06-08 17:30:36 UTC (rev 3751) @@ -472,7 +472,7 @@ spool_host, to_create)) return (GFARM_ERR_FILE_MIGRATED); if (to_create) { - e = inode_add_replica(fo->inode, spool_host); + e = inode_add_replica(fo->inode, spool_host, 1); if (e != GFARM_ERR_NO_ERROR) return (e); } @@ -667,6 +667,70 @@ (fo->flag & GFARM_FILE_CREATE) != 0, peer, spool_host, fdp)); } +gfarm_error_t +process_replica_adding(struct process *process, + struct peer *peer, struct host *spool_host, int fd, + gfarm_ino_t *inump, gfarm_uint64_t *genp, + gfarm_int64_t *mtime_secp, gfarm_int32_t *mtime_nsecp) +{ + struct file_opening *fo; + struct gfarm_timespec *mtime; + gfarm_error_t e = process_get_file_opening(process, fd, &fo); + + if (e != GFARM_ERR_NO_ERROR) + return (e); + if (!inode_is_file(fo->inode)) /* i.e. is a directory */ + return (GFARM_ERR_OPERATION_NOT_PERMITTED); + if (fo->u.f.spool_opener != NULL) /* already REOPENed */ + return (GFARM_ERR_OPERATION_NOT_PERMITTED); + if (inode_is_creating_file(fo->inode)) /* no file copy */ + return (GFARM_ERR_NO_SUCH_OBJECT); + if (inode_has_replica(fo->inode, spool_host)) + return (GFARM_ERR_ALREADY_EXISTS); + + e = inode_add_replica(fo->inode, spool_host, 0); + if (e != GFARM_ERR_NO_ERROR) + return (e); + + /* + * do not set spool_opener and spool_host. This file + * descriptor will be REOPENed by a source file system node. + */ + *inump = inode_get_number(fo->inode); + *genp = inode_get_gen(fo->inode); + mtime = inode_get_mtime(fo->inode); + *mtime_secp = mtime->tv_sec; + *mtime_nsecp = mtime->tv_nsec; + return (GFARM_ERR_NO_ERROR); +} + +gfarm_error_t +process_replica_added(struct process *process, + struct peer *peer, struct host *spool_host, int fd, + int flags, gfarm_int64_t mtime_sec, gfarm_int32_t mtime_nsec) +{ + struct file_opening *fo; + struct gfarm_timespec *mtime; + gfarm_error_t e = process_get_file_opening(process, fd, &fo); + + if (e != GFARM_ERR_NO_ERROR) + return (e); + if (!inode_is_file(fo->inode)) /* i.e. is a directory */ + return (GFARM_ERR_OPERATION_NOT_PERMITTED); + if (fo->u.f.spool_opener == NULL) /* not yet REOPENed */ + return (GFARM_ERR_OPERATION_NOT_PERMITTED); + if (inode_is_creating_file(fo->inode)) /* no file copy */ + return (GFARM_ERR_NO_SUCH_OBJECT); + if (inode_has_replica(fo->inode, spool_host)) + return (GFARM_ERR_ALREADY_EXISTS); + + mtime = inode_get_mtime(fo->inode); + if (mtime_sec != mtime->tv_sec || mtime_nsec != mtime->tv_nsec) + return (inode_remove_replica(fo->inode, spool_host)); + + return (inode_add_replica(fo->inode, spool_host, 1)); +} + /* * protocol handler */ Modified: gfarm/branches/gfarm_v2/server/gfmd/process.h =================================================================== --- gfarm/branches/gfarm_v2/server/gfmd/process.h 2007-06-07 01:06:48 UTC (rev 3750) +++ gfarm/branches/gfarm_v2/server/gfmd/process.h 2007-06-08 17:30:36 UTC (rev 3751) @@ -79,7 +79,7 @@ gfarm_error_t process_close_file_write(struct process *, struct peer *, int, gfarm_off_t, struct gfarm_timespec *, struct gfarm_timespec *); -gfarm_error_t process_cksum_set(struct process *, struct peer *, int , +gfarm_error_t process_cksum_set(struct process *, struct peer *, int, const char *, size_t, const char *, gfarm_int32_t, struct gfarm_timespec *); gfarm_error_t process_cksum_get(struct process *, struct peer *, int, @@ -93,3 +93,9 @@ gfarm_error_t gfm_server_bequeath_fd(struct peer *, int, int); gfarm_error_t gfm_server_inherit_fd(struct peer *, int, int); + +gfarm_error_t process_replica_adding(struct process *, struct peer *, + struct host *, int, gfarm_ino_t *, gfarm_uint64_t *, + gfarm_int64_t *, gfarm_int32_t *); +gfarm_error_t process_replica_added(struct process *, struct peer *, + struct host *, int, int, gfarm_int64_t, gfarm_int32_t); Modified: gfarm/branches/gfarm_v2/server/gfsd/gfsd.c =================================================================== --- gfarm/branches/gfarm_v2/server/gfsd/gfsd.c 2007-06-07 01:06:48 UTC (rev 3750) +++ gfarm/branches/gfarm_v2/server/gfsd/gfsd.c 2007-06-08 17:30:36 UTC (rev 3751) @@ -126,8 +126,8 @@ struct gfp_xdr *credential_exported = NULL; +long file_read_size; #if 0 /* not yet in gfarm v2 */ -long file_read_size; long rate_limit; #endif @@ -670,24 +670,17 @@ return (-1); } -gfarm_error_t -gfs_server_open_common(struct gfp_xdr *client, char *diag, - gfarm_int32_t *net_fdp, int *local_fdp) +static gfarm_error_t +gfs_server_reopen(char *diag, gfarm_int32_t net_fd, char **pathp, int *flagsp) { gfarm_error_t e; - gfarm_int32_t net_fd; gfarm_ino_t ino; gfarm_uint64_t gen; gfarm_int32_t mode, net_flags, to_create; char *path; - int local_fd, local_flags; + int local_flags; - gfs_server_get_request(client, diag, "i", &net_fd); - - if (!file_table_is_available(net_fd)) - e = GFARM_ERR_BAD_FILE_DESCRIPTOR; - - else if ((e = gfm_client_compound_begin_request(gfm_server)) + if ((e = gfm_client_compound_begin_request(gfm_server)) != GFARM_ERR_NO_ERROR) fatal_metadb_proto("compound_begin request", diag, e); else if ((e = gfm_client_put_fd_request(gfm_server, net_fd)) @@ -724,6 +717,28 @@ local_path(ino, gen, diag, &path); if (to_create) local_flags |= O_CREAT; + *pathp = path; + *flagsp = local_flags; + } + return (e); +} + +gfarm_error_t +gfs_server_open_common(struct gfp_xdr *client, char *diag, + gfarm_int32_t *net_fdp, int *local_fdp) +{ + gfarm_error_t e; + char *path; + int net_fd, local_fd, local_flags; + + gfs_server_get_request(client, diag, "i", &net_fd); + + if (!file_table_is_available(net_fd)) + e = GFARM_ERR_BAD_FILE_DESCRIPTOR; + else if ((e = gfs_server_reopen(diag, net_fd, &path, &local_flags)) + != GFARM_ERR_NO_ERROR) + ; + else { if ((local_fd = open_data(path, local_flags)) < 0) { e = gfarm_errno_to_error(errno); } else { @@ -1113,32 +1128,188 @@ "illllll", bsize, blocks, bfree, bavail, files, ffree, favail); } -#if 0 /* not yet in gfarm v2 */ +static gfarm_error_t +replica_adding(gfarm_int32_t net_fd, + char **pathp, gfarm_int64_t *mtime_secp, gfarm_int32_t *mtime_nsecp) +{ + gfarm_error_t e; + gfarm_ino_t ino; + gfarm_uint64_t gen; + gfarm_int64_t mtime_sec; + gfarm_int32_t mtime_nsec; + char *path, *diag = "replica_adding"; + if ((e = gfm_client_compound_begin_request(gfm_server)) + != GFARM_ERR_NO_ERROR) + fatal_metadb_proto("compound_begin request", diag, e); + else if ((e = gfm_client_put_fd_request(gfm_server, net_fd)) + != GFARM_ERR_NO_ERROR) + fatal_metadb_proto("put_fd request", diag, e); + else if ((e = gfm_client_replica_adding_request(gfm_server)) + != GFARM_ERR_NO_ERROR) + fatal_metadb_proto("replica_adding request", diag, e); + else if ((e = gfm_client_compound_end_request(gfm_server)) + != GFARM_ERR_NO_ERROR) + fatal_metadb_proto("compound_end request", diag, e); + + else if ((e = gfm_client_compound_begin_result(gfm_server)) + != GFARM_ERR_NO_ERROR) + fatal_metadb_proto("compound_begin result", diag, e); + else if ((e = gfm_client_put_fd_result(gfm_server)) + != GFARM_ERR_NO_ERROR) + fatal_metadb_proto("put_fd result", diag, e); + else if ((e = gfm_client_replica_adding_result(gfm_server, + &ino, &gen, &mtime_sec, &mtime_nsec)) + != GFARM_ERR_NO_ERROR) { + if (debug_mode) + gflog_info("replica_adding(%s) result: %s", diag, + gfarm_error_string(e)); + } else if ((e = gfm_client_compound_end_result(gfm_server)) + != GFARM_ERR_NO_ERROR) + fatal_metadb_proto("compound_end result", diag, e); + + else { + local_path(ino, gen, diag, &path); + *pathp = path; + *mtime_secp = mtime_sec; + *mtime_nsecp = mtime_nsec; + } + return (e); +} + +static gfarm_error_t +replica_added(gfarm_int32_t net_fd, + gfarm_int32_t flags, gfarm_int64_t mtime_sec, gfarm_int32_t mtime_nsec) +{ + gfarm_error_t e; + char *diag = "replica_added"; + + if ((e = gfm_client_compound_begin_request(gfm_server)) + != GFARM_ERR_NO_ERROR) + fatal_metadb_proto("compound_begin request", diag, e); + else if ((e = gfm_client_put_fd_request(gfm_server, net_fd)) + != GFARM_ERR_NO_ERROR) + fatal_metadb_proto("put_fd request", diag, e); + else if ((e = gfm_client_replica_added_request(gfm_server, + flags, mtime_sec, mtime_nsec)) + != GFARM_ERR_NO_ERROR) + fatal_metadb_proto("replica_added request", diag, e); + else if ((e = gfm_client_compound_end_request(gfm_server)) + != GFARM_ERR_NO_ERROR) + fatal_metadb_proto("compound_end request", diag, e); + + else if ((e = gfm_client_compound_begin_result(gfm_server)) + != GFARM_ERR_NO_ERROR) + fatal_metadb_proto("compound_begin result", diag, e); + else if ((e = gfm_client_put_fd_result(gfm_server)) + != GFARM_ERR_NO_ERROR) + fatal_metadb_proto("put_fd result", diag, e); + else if ((e = gfm_client_replica_added_result(gfm_server)) + != GFARM_ERR_NO_ERROR) { + if (debug_mode) + gflog_info("replica_added(%s) result: %s", diag, + gfarm_error_string(e)); + } else if ((e = gfm_client_compound_end_result(gfm_server)) + != GFARM_ERR_NO_ERROR) + fatal_metadb_proto("compound_end result", diag, e); + + return (e); +} + void -gfs_server_bulkread(struct gfp_xdr *client) +gfs_server_replica_add_from(struct gfp_xdr *client) { + gfarm_int32_t net_fd, local_fd, port, mtime_nsec; + gfarm_int64_t mtime_sec; gfarm_error_t e; + char *host, *path, *diag = "replica_add_from"; + struct gfs_connection *server; + int flags = 0; /* XXX - for now */ + gfarm_pid_t pid; + gfarm_int32_t keytype; + size_t keylen; + char sharedkey[GFM_PROTO_PROCESS_KEY_LEN_SHAREDSECRET]; + + gfs_server_get_request(client, diag, "iblsii", + &keytype, sizeof(sharedkey), &keylen, sharedkey, &pid, + &host, &port, &net_fd); + + e = gfm_client_process_set(gfm_server, + keytype, sharedkey, keylen, pid); + if (e != GFARM_ERR_NO_ERROR) + goto free_host; + + e = replica_adding(net_fd, &path, &mtime_sec, &mtime_nsec); + if (e != GFARM_ERR_NO_ERROR) + goto free_host; + + local_fd = open_data(path, O_WRONLY|O_CREAT|O_TRUNC); + free(path); + if (local_fd < 0) { + e = gfarm_errno_to_error(errno); + goto free_host; + } + + e = gfs_client_connection_acquire_by_host(host, port, &server); + if (e != GFARM_ERR_NO_ERROR) + goto close; + e = gfs_client_process_set(server, keytype, keylen, sharedkey, pid); + if (e != GFARM_ERR_NO_ERROR) + goto connection_free; + e = gfs_client_replica_recv(server, net_fd, local_fd); + if (e != GFARM_ERR_NO_ERROR) + goto connection_free; + e = replica_added(net_fd, flags, mtime_sec, mtime_nsec); + + connection_free: + gfs_client_connection_free(server); + close: + close(local_fd); + free_host: + free(host); + gfs_server_put_reply(client, diag, e, ""); + return; +} + +void +gfs_server_replica_recv(struct gfp_xdr *client) +{ + gfarm_error_t e, error = GFARM_ERR_NO_ERROR; gfarm_int32_t fd; ssize_t rv; - gfarm_error_t error = GFARM_ERR_NO_ERROR; char buffer[GFS_PROTO_MAX_IOSIZE]; +#if 0 /* not yet in gfarm v2 */ struct gfs_client_rep_rate_info *rinfo = NULL; +#endif + char *diag = "replica_recv", *path; + int local_fd, local_flags; - gfs_server_get_request(client, "bulkread", "i", &fd); + gfs_server_get_request(client, diag, "i", &fd); + error = gfs_server_reopen(diag, fd, &path, &local_flags); + if (error != GFARM_ERR_NO_ERROR) + goto send_eof; + + local_fd = open_data(path, local_flags); + free(path); + if (local_fd < 0) { + error = gfarm_errno_to_error(errno); + goto send_eof; + } + + /* data transfer */ if (file_read_size >= sizeof(buffer)) file_read_size = sizeof(buffer); +#if 0 /* not yet in gfarm v2 */ if (rate_limit != 0) { rinfo = gfs_client_rep_rate_info_alloc(rate_limit); if (rinfo == NULL) - fatal("bulkread:rate_info_alloc: %s", + fatal("%s:rate_info_alloc: %s", diag, gfarm_error_string(GFARM_ERR_NO_MEMORY)); } - - fd = file_table_get(fd); +#endif do { - rv = read(fd, buffer, file_read_size); + rv = read(local_fd, buffer, file_read_size); if (rv <= 0) { if (rv == -1) error = gfarm_errno_to_error(errno); @@ -1156,20 +1327,30 @@ break; } } +#if 0 /* not yet in gfarm v2 */ if (rate_limit != 0) gfs_client_rep_rate_control(rinfo, rv); +#endif } while (rv > 0); +#if 0 /* not yet in gfarm v2 */ if (rinfo != NULL) gfs_client_rep_rate_info_free(rinfo); +#endif + e = close(local_fd); + if (error == GFARM_ERR_NO_ERROR) + error = e; + send_eof: /* send EOF mark */ e = gfp_xdr_send(client, "b", 0, buffer); - if (e != GFARM_ERR_NO_ERROR && error == GFARM_ERR_NO_ERROR) + if (error == GFARM_ERR_NO_ERROR) error = e; - gfs_server_put_reply(client, "bulkread", error, ""); + gfs_server_put_reply(client, diag, error, ""); } +#if 0 /* not yet in gfarm v2 */ + void gfs_server_striping_read(struct gfp_xdr *client) { @@ -2610,6 +2791,8 @@ client_name, client_addr, &rate_limit); if (e != GFARM_ERR_NO_ERROR) /* shouldn't happen */ fatal("rate_limit: %s", gfarm_error_string(e)); +#else + file_read_size = GFS_PROTO_MAX_IOSIZE; #endif /* not yet in gfarm v2 */ e = gfp_xdr_new_socket(client_fd, &client); @@ -2679,6 +2862,10 @@ gfp_xdr_env_for_credential(client)); break; #endif /* not yet in gfarm v2 */ + case GFS_PROTO_REPLICA_ADD_FROM: + gfs_server_replica_add_from(client); break; + case GFS_PROTO_REPLICA_RECV: + gfs_server_replica_recv(client); break; default: gflog_warning("unknown request %d", (int)request); cleanup(0); This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |