diff --git a/ctr/migration.go b/ctr/migration.go index c288139..0603224 100644 --- a/ctr/migration.go +++ b/ctr/migration.go @@ -12,7 +12,7 @@ var migrationCommand = cli.Command{ Name: "migration", Usage: "migration containers", ArgsUsage: "-H -p -A ||" + - " 192.168.18.2 9001 -A -e MYSQL_ROOT_PASSWORD=123456 mysql", + " 192.168.18.2 9001 -A -e,MYSQL_ROOT_PASSWORD=123456 mysql", Flags: []cli.Flag{ cli.StringFlag{ Name: "host,H", diff --git a/supervisor/migration.go b/supervisor/migration.go index 3ed23f9..f96d558 100644 --- a/supervisor/migration.go +++ b/supervisor/migration.go @@ -6,14 +6,20 @@ import ( "github.com/containerd/containerd/api/grpc/types" "github.com/containerd/containerd/runtime" "github.com/containerd/containerd/supervisor/migration" - "github.com/sirupsen/logrus" - "log" + "io" "net" "os" "strings" "time" + "log" +) + +const ( + TimeLogPos = "/run/migration/time.log" + TimeLogPath="/run/migration" ) + // type MigrationTask struct { baseTask @@ -27,55 +33,57 @@ type TargetMachine struct { Port uint32 } -var ( - TimeLogger *log.Logger - TimeLogPos = "/run/migration/time.log" -) +type doubleLoger struct { + w io.Writer +} -func (s *Supervisor) StartMigration(t *MigrationTask) error { - logrus.SetFormatter(&logrus.TextFormatter{FullTimestamp:true}) +func (d *doubleLoger) Write(p []byte) (n int, err error) { + os.Stderr.Write(p) + return d.w.Write(p) +} + +func NewDoubleLoger(f io.Writer) io.Writer { + return &doubleLoger{w: f} +} + +func init() { + + log.SetFlags(log.Lmicroseconds|log.Lshortfile) + log.SetPrefix("Migration log:") + os.MkdirAll(TimeLogPath, 0755) f, err := os.OpenFile(TimeLogPos, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) if err != nil { - logrus.Printf("set timelog err:%v\n", err) - return err + log.Printf("set timelog err:%v\n", err) + panic(err) } + //log.SetOutput(NewDoubleLoger(f)) + log.SetOutput(NewDoubleLoger(f)) +} - TimeLogger = log.New(f, "TimeLog: ", log.LUTC|log.Lshortfile) - - defer f.Close() - - //c, err := t.checkContainers(s) - //if err != nil { - // logrus.Println(err) - // return err - //} - // - //if err = t.checkTargetMachine(); err != nil { - // return err - //} +func (s *Supervisor) StartMigration(t *MigrationTask) error { c, _, err := t.PerMigrationTask(s) if err != nil { - logrus.Printf("premigration faild:%v\n", err) + log.Printf("premigration faild:%v\n", err) return err } if err = t.startMigrationTask(c); err != nil { - logrus.Println("start error: ", err) + log.Println("start error: ", err) return err } - logrus.Println("migration Finish") + log.Println("migration Finish") return nil } func (t *MigrationTask) PerMigrationTask(s *Supervisor) (*containerInfo, types.APIClient, error) { start := time.Now() - TimeLogger.Printf("start premigration task at %v\n", start.String()) + log.Printf("start premigration task at %v\n", start.String()) c, err := t.checkContainers(s) if err != nil { - logrus.Println(err) + log.Println(err) return nil, nil, err } @@ -85,14 +93,14 @@ func (t *MigrationTask) PerMigrationTask(s *Supervisor) (*containerInfo, types.A } end := time.Now() - TimeLogger.Printf("end premigration task at %v\n", end.String()) - TimeLogger.Printf("pre migration task cost %v\n", end.Sub(start).String()) + log.Printf("end premigration task at %v\n", end.String()) + log.Printf("pre migration task cost %v\n", end.Sub(start).String()) return c, rpclient, nil } func (t *MigrationTask) checkContainers(s *Supervisor) (*containerInfo, error) { - logrus.Println("check containers exist") + log.Println("check containers exist") i, ok := s.containers[t.Id] if !ok { @@ -106,7 +114,7 @@ func (t *MigrationTask) checkContainers(s *Supervisor) (*containerInfo, error) { func (t *MigrationTask) checkTargetMachine() (types.APIClient, error) { - logrus.Println("check target machine") + log.Println("check target machine") ip := t.Host port := t.Port @@ -132,36 +140,36 @@ func (t *MigrationTask) checkTargetMachine() (types.APIClient, error) { func (t *MigrationTask) startMigrationTask(c *containerInfo) error { start := time.Now() - TimeLogger.Printf("start migration task at %v\n", start.String()) + log.Printf("start migration task at %v\n", start.String()) var ( e = make(chan error) err error ) - logrus.Println("new local") + log.Println("new local") l, err := migration.NewLocalMigration(c.container) if err != nil { return MigrationWriteErr(err.Error()) } - logrus.Println("new remote") + log.Println("new remote") r, err := migration.NewRemoteMigration(t.Host, t.Id, t.Port) if err != nil { return MigrationWriteErr(err.Error()) } - logrus.Println("start preload image in goroutine") + log.Println("start preload image in goroutine") go r.PreLoadImage(e, l.Imagedir) //TODO 将hostpath的目录nfs到远程挂载 准备在本机的工作 - logrus.Println("start nfs hostpath") + log.Println("start nfs hostpath") if err = l.SetNfsExport(); err != nil { - logrus.Println("nfs", err) + log.Println("nfs", err) return err } - logrus.Println("set spec") + log.Println("set spec") if err = r.SetSpec(l); err != nil { return err } @@ -169,9 +177,9 @@ func (t *MigrationTask) startMigrationTask(c *containerInfo) error { if err = <-e; err != nil { return MigrationWriteErr(err.Error()) } - logrus.Println("wait goroutines finish") + log.Println("wait goroutines finish") - logrus.Println("do checkpoint") + log.Println("do checkpoint") if err = l.DoCheckpoint(); err != nil { return err } @@ -182,31 +190,31 @@ func (t *MigrationTask) startMigrationTask(c *containerInfo) error { return err } - logrus.Println("copy checkpoint dir") + log.Println("copy checkpoint dir") if err = l.CopyCheckPointToRemote(r); err != nil { return err } - logrus.Println("copy upperdir") + log.Println("copy upperdir") if err = l.CopyUpperToRemote(r); err != nil { return err } //在目的主机进行premigration准备操作 - logrus.Println("start premigration") + log.Println("start premigration") if err = r.PreRemoteMigration(t.Id, l.Imagedir.GetUpperId(), t.Args); err != nil { - logrus.Printf("premigration error: %v\n", err) + log.Printf("premigration error: %v\n", err) return err } if err = r.DoRestore(); err != nil { return MigrationWriteErr(err.Error()) } - logrus.Println("done restore") + log.Println("done restore") end := time.Now() - TimeLogger.Printf("end migration task ai %v\n", end.String()) - TimeLogger.Printf("migration task cost %v\n", end.Sub(start)) + log.Printf("end migration task ai %v\n", end.String()) + log.Printf("migration task cost %v\n", end.Sub(start)) return nil } @@ -272,10 +280,10 @@ func MigrationWriteErr(w string) error { // e.Stderr="/dev/null" // e.BundlePath=r.Bundle // e.StartResponse = make(chan StartResponse, 1) -// logrus.Println("restore send task") +// log.Println("restore send task") // s.SendTask(e) // if err := <-e.ErrorCh(); err != nil { -// logrus.Println(err) +// log.Println(err) // return err // } // <-e.StartResponse diff --git a/supervisor/migration/containerUtils.go b/supervisor/migration/containerUtils.go index 758b5b8..1823ae2 100644 --- a/supervisor/migration/containerUtils.go +++ b/supervisor/migration/containerUtils.go @@ -22,13 +22,13 @@ import ( "time" ) -var ( - glog *log.Logger -) - -func init() { - glog = log.New(os.Stderr, "", log.Lshortfile) -} +//var ( +// log *log.Logger +//) +// +//func init() { +// log = log.New(os.Stderr, "", log.Lshortfile) +//} func LoadSpec(c runtime.Container) (*specs.Spec, error) { var spec specs.Spec @@ -107,7 +107,7 @@ func RemoteCopyDir(localPath, remotePath string, c *sftp.Client) error { ) if _, err = c.Stat(remotePath); err == nil { - glog.Printf("remote has %v,we do not copy it", remotePath) + log.Printf("remote has %v,we do not copy it", remotePath) return nil } @@ -131,7 +131,7 @@ func RemoteCopyDir(localPath, remotePath string, c *sftp.Client) error { if err = c.Mkdir(rpath); err != nil { //panic(err) - glog.Println(err) + log.Println(err) return err } @@ -151,14 +151,14 @@ func RemoteCopyDir(localPath, remotePath string, c *sftp.Client) error { _, err = io.CopyBuffer(dstf, srcf, buf) if err != nil { //panic(err) - glog.Println(err) + log.Println(err) return err } } return nil }) if err != nil { - glog.Println(err) + log.Println(err) return err } return nil @@ -177,13 +177,13 @@ func RemoteCopyDirRsync(local, remote string, ip string) error { } args := append([]string{"-azv"}, local, "root@"+ip+":"+remote) - //glog.Printf("l is %v,r is %v,args is %v\n",local,remote,args) + //log.Printf("l is %v,r is %v,args is %v\n",local,remote,args) cmd := exec.Command("rsync", args...) - //glog.Printf("cmd is %v\n",cmd) + //log.Printf("cmd is %v\n",cmd) if err = cmd.Run(); err != nil { - glog.Printf("rsync error:%v\n", err) - glog.Printf("cmd is %v\n", cmd.Args) + log.Printf("rsync error:%v\n", err) + log.Printf("cmd is %v\n", cmd.Args) } return err } @@ -197,10 +197,10 @@ func CopyDirLocal(src, dst string) error { args := append([]string{"-azv"}, local, remote) cmd := exec.Command("rsync", args...) - //glog.Printf("cmd is %v\n",cmd) + //log.Printf("cmd is %v\n",cmd) if err = cmd.Run(); err != nil { - glog.Printf("rsync error:%v\n", err) - glog.Printf("cmd is %v\n", cmd.Args) + log.Printf("rsync error:%v\n", err) + log.Printf("cmd is %v\n", cmd.Args) } return err @@ -216,12 +216,12 @@ func RemoteMkdirAll(rpath string, c *sftp.Client) error { if _, err := c.Stat(root); err != nil { if err == os.ErrNotExist { - //glog.Printf("dir %v not exist,we create it\n",root) + //log.Printf("dir %v not exist,we create it\n",root) if err := c.Mkdir(root); err != nil { return err } } else { - glog.Println(err) + log.Println(err) return err } } @@ -245,25 +245,25 @@ func GetVolume(id string) ([]Volumes, error) { bs, err := cmd.Output() if err != nil { - glog.Println(err) + log.Println(err) return nil, err } if err = json.Unmarshal(bs, &vl); err != nil { - glog.Println(err) + log.Println(err) return nil, err } if len(vl) != 1 { - glog.Println("len !=1 ") + log.Println("len !=1 ") return nil, errors.New("inspect not one\n") } for _, v := range vl[0].HostConfig.Binds { sp := strings.Split(v, ":") if len(sp) != 2 { - glog.Println("splite false") - glog.Println(sp) + log.Println("splite false") + log.Println(sp) return nil, errors.New("split error\n") } res = append(res, struct{ src, dst string }{src: sp[0], dst: sp[1]}) @@ -281,12 +281,12 @@ func GetCName(id string) (string, error) { cmd := exec.Command("docker", args...) bs, err := cmd.Output() if err != nil { - glog.Println(err) + log.Println(err) return "", err } if err = json.Unmarshal(bs, &tmp); err != nil { - glog.Println(err) + log.Println(err) return "", err } name := tmp[0].Name @@ -310,12 +310,12 @@ func GetImage(id string) (string, error) { bs, err := cmd.Output() if err != nil { - glog.Println(err) + log.Println(err) return res, err } if err = json.Unmarshal(bs, &tmp); err != nil { - glog.Println(err) + log.Println(err) return res, err } @@ -328,14 +328,14 @@ func SetNfsExport(vol []Volumes) error { f, err := os.OpenFile("/etc/exports", os.O_RDWR|os.O_APPEND, 0666) if err != nil { - glog.Println(err) + log.Println(err) return err } defer f.Close() for _, v := range vol { if _, err = fmt.Fprintf(f, "%s %s\n", v.src, nfsconfig); err != nil { - glog.Println(err) + log.Println(err) return err } } @@ -343,7 +343,7 @@ func SetNfsExport(vol []Volumes) error { f.Sync() if err = FlushNfsConfig(); err != nil { - glog.Println(err) + log.Println(err) return err } return nil @@ -356,14 +356,14 @@ func GetIp() (string, error) { ) addrs, err := net.InterfaceAddrs() if err != nil { - glog.Println(err) + log.Println(err) return "", err } for _, address := range addrs { if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { if ipnet.IP.To4() != nil { - //glog.Printf("ip is %v\n", ipnet.IP.String()) + //log.Printf("ip is %v\n", ipnet.IP.String()) return ipnet.IP.String(), nil } } diff --git a/supervisor/migration/image.go b/supervisor/migration/image.go index 7e5d7e5..885a975 100644 --- a/supervisor/migration/image.go +++ b/supervisor/migration/image.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "strings" + "log" ) const Driver = "overlay2" @@ -110,18 +111,18 @@ func (i *Image) CopyUpper(c *sftp.Client) error { mudir := i.upperRD[:len(i.upperRD)-5] remoteDir, err := PathToRemote(mudir) if err != nil { - glog.Println(err) + log.Println(err) return err } - glog.Printf("copy upper dir %v\n", mudir) + log.Printf("copy upper dir %v\n", mudir) if err = RemoteCopyDir(filepath.Join(mudir, "diff"), filepath.Join(remoteDir, "diff"), c); err != nil { - glog.Println(err) + log.Println(err) return err } if err = RemoteCopyDir(filepath.Join(mudir, "lower"), filepath.Join(remoteDir, "lower"), c); err != nil { - glog.Println(err) + log.Println(err) return err } return nil @@ -135,7 +136,7 @@ func (i *Image) PreCopyImage(c *sftp.Client, r *remoteMigration) error { if err != nil { return err } - //glog.Printf("v :%v,r %v\n", v, remotePath) + //log.Printf("v :%v,r %v\n", v, remotePath) _, err = c.Stat(remotePath) if err != nil { //TODO 远程不存在该文件，则传输过去 @@ -151,7 +152,7 @@ func (i *Image) PreCopyImage(c *sftp.Client, r *remoteMigration) error { return err } } - //glog.Printf("remote has dir,so not copy:%v\n",w.Name()) + //log.Printf("remote has dir,so not copy:%v\n",w.Name()) } return nil diff --git a/supervisor/migration/lazycopydir/JobList.go b/supervisor/migration/lazycopydir/JobList.go index a229be3..e682767 100644 --- a/supervisor/migration/lazycopydir/JobList.go +++ b/supervisor/migration/lazycopydir/JobList.go @@ -4,6 +4,7 @@ import ( "errors" "strings" "sync" + "log" ) var ( @@ -55,7 +56,7 @@ func (l *JobList) Remove(v string) error { return JobListRemoveEmpty } - glog.Printf("list remove file %v\n", v) + log.Printf("list remove file %v\n", v) if len(l.data) == 1 { if l.data[0] == v { @@ -88,10 +89,10 @@ func (l *JobList) RemoveAllDir(v string) error { l.w.Lock() defer l.w.Unlock() - if v[len(v)-1] != '\\' { - glog.Printf("remove dir error:%v is not a dir\n", v) + if v[len(v)-1] != '/' { + log.Printf("remove dir error: %v is not a dir\n", v) } - glog.Printf("list remove dir %v \n", v) + log.Printf("list remove dir %v \n", v) if len(l.data) == 0 { return JobListRemoveEmpty } diff --git a/supervisor/migration/lazycopydir/crawler_test.go b/supervisor/migration/lazycopydir/crawler_test.go index 0f6db88..e526c38 100644 --- a/supervisor/migration/lazycopydir/crawler_test.go +++ b/supervisor/migration/lazycopydir/crawler_test.go @@ -11,12 +11,12 @@ import ( func TestCrawlerAllFiles(t *testing.T) { l := NewJobList() - dir := "/var/lib/docker/workfile/overlaytest/lower" + dir := "/var/lib/docker/workfile/vols/data" if err := CrawlerAllFiles(dir, l); err != nil { t.Error(err) return } - l.Remove(".") + //l.Remove(".") fmt.Printf("l len is %d\n", len(l.data)) v, err := l.Pop() for err == nil { diff --git a/supervisor/migration/lazycopydir/crawlerdir.go b/supervisor/migration/lazycopydir/crawlerdir.go index 1cf9106..d06ed0a 100644 --- a/supervisor/migration/lazycopydir/crawlerdir.go +++ b/supervisor/migration/lazycopydir/crawlerdir.go @@ -7,18 +7,18 @@ import ( "path/filepath" ) -var ( - glog *log.Logger -) - -func init() { - glog = log.New(os.Stderr, "lazyCopyLog: ", log.Lshortfile) -} +//var ( +// log *log.Logger +//) +// +//func init() { +// log = log.New(os.Stderr, "lazyCopyLog: ", log.Lshortfile) +//} func (l *LazyReplicator) Crawler() error { if len(l.List.data) > 0 { - glog.Println("crawler should start with empty joblist") + log.Println("crawler should start with empty joblist") _, err := l.List.Pop() for err != JobListPopError { _, err = l.List.Pop() @@ -36,18 +36,18 @@ func CrawlerAllFiles(dir string, list *JobList) error { ) if tmpdir, err = os.Getwd(); err != nil { - glog.Println(err) + log.Println(err) return err } defer func() { if err = os.Chdir(tmpdir); err != nil { - glog.Fatalln(err) + log.Fatalln(err) } }() if err = os.Chdir(dir); err != nil { - glog.Println(err) + log.Println(err) return err } @@ -55,7 +55,7 @@ func CrawlerAllFiles(dir string, list *JobList) error { if err = filepath.Walk(".", func(path string, info os.FileInfo, err error) error { if err != nil { - glog.Println(err) + log.Println(err) return err } @@ -66,8 +66,9 @@ func CrawlerAllFiles(dir string, list *JobList) error { } return nil }); err != nil { - glog.Println(err) + log.Println(err) return err } + list.Pop() return nil } diff --git a/supervisor/migration/lazycopydir/manager.go b/supervisor/migration/lazycopydir/manager.go index 35978a7..0828087 100644 --- a/supervisor/migration/lazycopydir/manager.go +++ b/supervisor/migration/lazycopydir/manager.go @@ -1,6 +1,9 @@ package lazycopydir -import "time" +import ( + "time" + "log" +) //crwdir 挂载的nfs目录 monidiruper读写层目录 lazydir 惰复制目录 func StartLazyCopy(crwdir, monidir, lazydir string) error { @@ -12,20 +15,20 @@ func StartLazyCopy(crwdir, monidir, lazydir string) error { replicator := NewLazyReplicator(crwdir, monidir, lazydir) if err = replicator.Crawler(); err != nil { - glog.Println(err) + log.Println(err) return err } go func() { err = replicator.Monitor() if err != nil { - glog.Println(err) + log.Println(err) panic(err) } }() if err = replicator.Replicate(); err != nil { - glog.Println(err) + log.Println(err) panic(err) } @@ -33,7 +36,7 @@ func StartLazyCopy(crwdir, monidir, lazydir string) error { time.Sleep(1 * time.Second) - glog.Println("finish lazycopy!") + log.Println("finish lazycopy!") return nil } @@ -43,14 +46,18 @@ func (replicator *LazyReplicator) Prelazy() error { ) if err = replicator.Crawler(); err != nil { - glog.Println(err) + log.Println(err) return err } - + log.Printf("crawler %v finish,len is %v\n",replicator.CrawlerDir,len(replicator.List.data)) + for _,v:=range replicator.List.data { + log.Println(v) + } + log.Println(" ") go func() { err = replicator.Monitor() if err != nil { - glog.Println(err) + log.Println(err) panic(err) } }() @@ -64,7 +71,7 @@ func (replicator *LazyReplicator) Dolazycopy() error { ) if err = replicator.Replicate(); err != nil { - glog.Println(err) + log.Println(err) panic(err) } @@ -72,7 +79,7 @@ func (replicator *LazyReplicator) Dolazycopy() error { time.Sleep(1 * time.Second) - glog.Println("finish lazycopy!") + log.Println("finish lazycopy!") return nil } diff --git a/supervisor/migration/lazycopydir/monitor.go b/supervisor/migration/lazycopydir/monitor.go index 88fc2a5..916b5f4 100644 --- a/supervisor/migration/lazycopydir/monitor.go +++ b/supervisor/migration/lazycopydir/monitor.go @@ -5,6 +5,7 @@ import ( "github.com/fsnotify/fsnotify" "os" "path/filepath" + "log" ) func (l *LazyReplicator) Monitor() error { @@ -19,28 +20,28 @@ func MonitorDir(dir string, list *JobList, ctx context.Context, crawdir string) ) if w, err = fsnotify.NewWatcher(); err != nil { - glog.Println(err) + log.Println(err) return err } //monitor monidir,remove updated file from list if err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { - glog.Println(err) + log.Println(err) return err } //add all dir to watch list if info.IsDir() { if err = w.Add(path); err != nil { - glog.Println(err) + log.Println(err) return err } - glog.Printf("add %v to monitor lists\n", path) + log.Printf("add %v to monitor lists\n", path) } return nil }); err != nil { - glog.Println(err) + log.Println(err) return err } @@ -53,12 +54,12 @@ func MonitorDir(dir string, list *JobList, ctx context.Context, crawdir string) info, err := os.Stat(events.Name) if err == nil && info.IsDir() { if err = w.Add(events.Name); err != nil { - glog.Println(err) + log.Println(err) } } if addpath, err = filepath.Rel(dir, events.Name); err != nil { - glog.Printf("Rel err:%v\n", err) + log.Printf("Rel err:%v\n", err) } else { if isDir(addpath, crawdir) { @@ -69,10 +70,10 @@ func MonitorDir(dir string, list *JobList, ctx context.Context, crawdir string) } else { if err = list.Remove(addpath); err != nil { - glog.Println(err) + log.Println(err) } } - + log.Printf("remove %v,now len is %v\n",addpath,len(list.data)) } } @@ -81,7 +82,7 @@ func MonitorDir(dir string, list *JobList, ctx context.Context, crawdir string) info, err := os.Stat(events.Name) if err == nil && info.IsDir() { if err = w.Remove(events.Name); err != nil { - glog.Println(err) + log.Println(err) } } @@ -90,10 +91,10 @@ func MonitorDir(dir string, list *JobList, ctx context.Context, crawdir string) case err = <-w.Errors: if err != nil { //只打印出错误，继续监视 - glog.Println(err) + log.Println(err) } case <-ctx.Done(): - glog.Println("Exist Monitor") + log.Println("Exist Monitor") goto End } } @@ -101,7 +102,7 @@ func MonitorDir(dir string, list *JobList, ctx context.Context, crawdir string) goto End End: - glog.Println("Monitor End") + log.Println("Monitor End") return w.Close() } diff --git a/supervisor/migration/lazycopydir/replicator.go b/supervisor/migration/lazycopydir/replicator.go index 3eb7650..98d0cb2 100644 --- a/supervisor/migration/lazycopydir/replicator.go +++ b/supervisor/migration/lazycopydir/replicator.go @@ -6,6 +6,7 @@ import ( "io" "os" "path/filepath" + "log" ) type LazyReplicator struct { @@ -45,8 +46,8 @@ func (l *LazyReplicator) Replicate() error { ) file, err = l.List.Pop() + log.Printf("first file is %v,len is %v\n", file, len(l.List.data)) for err == nil { - sourcedir = filepath.Join(l.CrawlerDir, file) targetdir = filepath.Join(l.LazyDir, file) @@ -57,11 +58,11 @@ func (l *LazyReplicator) Replicate() error { } } else { if err = LocalCopy(sourcedir, targetdir); err != nil { - glog.Println(err) + log.Println(err) } } - file, err = l.List.Pop() + log.Printf("next file is %v,len is %v\n", file, len(l.List.data)) } return nil } @@ -80,22 +81,22 @@ func LocalCopy(source, target string) error { ) if _, err = os.Stat(target); err == nil { - glog.Printf("target %v exist,don't copy\n", target) - return nil + log.Printf("target %v exist,don't copy\n", target) + return err } if !os.IsNotExist(err) { return fmt.Errorf("Copy Error:%v\n", err) } if err = os.MkdirAll(filepath.Dir(target), 0755); err != nil { - glog.Println(err) + log.Println(err) return err } if src, err = os.Open(source); err != nil { - glog.Println(err) + log.Println(err) return err } if dst, err = os.Create(target); err != nil { - glog.Println(err) + log.Println(err) return err } defer func() { @@ -104,5 +105,6 @@ func LocalCopy(source, target string) error { }() _, err = io.Copy(dst, src) + log.Printf("now copy %v\n", target) return err } diff --git a/supervisor/migration/localMigration.go b/supervisor/migration/localMigration.go index 81d7aa4..96e90b0 100644 --- a/supervisor/migration/localMigration.go +++ b/supervisor/migration/localMigration.go @@ -7,6 +7,7 @@ import ( //"github.com/containerd/containerd/supervisor" "errors" "fmt" + "log" ) const MigrationDir = "/run/migration" @@ -72,7 +73,7 @@ func (l *localMigration) DoCheckpoint() error { if err = os.RemoveAll(ldir); err != nil { return err } - glog.Println("checkpoint dir exist,we remove it") + log.Println("checkpoint dir exist,we remove it") } return l.Checkpoint(doCheckpoint, l.CheckpointDir) @@ -99,7 +100,7 @@ func (l *localMigration) CopyUpperToRemote(r *remoteMigration) error { } if err = RemoteMkdirAll(remoteUpperDir, r.sftpClient); err != nil { - glog.Println(err) + log.Println(err) return err } @@ -115,17 +116,18 @@ func (l *localMigration) CopyCheckPointToRemote(r *remoteMigration) error { return fmt.Errorf("Err: remote nil\n ") } + if err := RemoteMkdirAll(r.CheckpointDir, r.sftpClient); err != nil { - glog.Println(err) + log.Println(err) return err } if err := RemoteCopyDirRsync(l.CheckpointDir, r.CheckpointDir, r.ip); err != nil { - glog.Println(err) + log.Println(err) return err } //if err := RemoteCopyDir(l.CheckpointDir, r.CheckpointDir, r.sftpClient); err != nil { - // glog.Println(err) + // log.Println(err) // return err //} return nil @@ -142,24 +144,24 @@ func (l *localMigration) CopyCheckPointToRemote(r *remoteMigration) error { // return false, err // } // if len(spec.Mounts) == 0 { -// glog.Println(spec.Mounts) +// log.Println(spec.Mounts) // return false, errors.New("nfs error: no mounts\n") // } // // //f,err:=os.Open("/etc/export") // f, err := os.OpenFile("/etc/exports", os.O_RDWR|os.O_APPEND, 0666) // if err != nil { -// glog.Println(err) +// log.Println(err) // return true, err // } // defer f.Close() // // for _, v := range spec.Mounts { -// glog.Printf("type is %v,optionsis %v,dst is %v\n", v.Type, v.Options, v.Destination) +// log.Printf("type is %v,optionsis %v,dst is %v\n", v.Type, v.Options, v.Destination) // if v.Type == "bind" && len(v.Options) == 1 && v.Options[0] == "rbind" { // count++ // if _, err = fmt.Fprintf(f, "%s %s", v.Source, nfsconfig); err != nil { -// glog.Println(err) +// log.Println(err) // return true, err // } // } @@ -178,14 +180,14 @@ func (l *localMigration) SetNfsExport() error { vol, err := GetVolume(l.ID()) if err != nil { - glog.Println(err) + log.Println(err) return err } if len(vol) == 0 { return nil } if err = SetNfsExport(vol); err != nil { - glog.Println(err) + log.Println(err) } return err diff --git a/supervisor/migration/preMigration.go b/supervisor/migration/preMigration.go index 889ad46..2dd2e6a 100644 --- a/supervisor/migration/preMigration.go +++ b/supervisor/migration/preMigration.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strconv" "time" + "log" ) const preVolume = "/var/lib/migration/mvolume" @@ -38,58 +39,58 @@ func (p *PreMigrationInTargetMachine) StartPre() error { goto CREATCONT } - glog.Println("premkdir") + log.Println("premkdir") if err = p.PreMkVolDir(); err != nil { - glog.Println(err) + log.Println(err) return err } CREATCONT: - glog.Println("create docker container") + log.Println("create docker container") if err = p.CreateDockerContainer(); err != nil { - glog.Println(err) + log.Println(err) return err } - glog.Println("copy upperdir") + log.Println("copy upperdir") if err = p.CopyUpperDir(p.UpperId); err != nil { - glog.Println(err) + log.Println(err) return err } if len(p.Vol) == 0 { goto STARTCONT } - glog.Println("mount nfs") + log.Println("mount nfs") if err = p.MountNfs(); err != nil { - glog.Println(err) + log.Println(err) return err } - glog.Println("start overlay dir") + log.Println("start overlay dir") if err = p.PreLazyDir(); err != nil { - glog.Println(err) + log.Println(err) return err } - glog.Println("pre lazycopy") + log.Println("pre lazycopy") if err = p.StartPreLazyCopy(); err != nil { - glog.Println(err) + log.Println(err) return err } STARTCONT: - glog.Println("start docker container") + log.Println("start docker container") if err = p.StartDockerContainer(); err != nil { - glog.Println(err) + log.Println(err) return err } - glog.Printf("now container start run! %v\n", time.Now()) + log.Printf("now container start run! %v\n", time.Now()) - glog.Println("start lazycopy") + log.Println("start lazycopy") if err = p.StartLazyCopy(); err != nil { - glog.Println(err) + log.Println(err) return err } return nil @@ -99,26 +100,27 @@ func (p *PreMigrationInTargetMachine) PreMkVolDir() error { var ( err error ) + os.RemoveAll(filepath.Join(preVolume,p.Id)) for i := 0; i < len(p.Vol); i++ { tpath := filepath.Join(preVolume, p.Id, strconv.Itoa(i)) if err = os.MkdirAll(filepath.Join(tpath, "lazy"), 0755); err != nil { - glog.Println(err) + log.Println(err) return err } if err = os.MkdirAll(filepath.Join(tpath, "upper"), 0755); err != nil { - glog.Println(err) + log.Println(err) return err } if err = os.MkdirAll(filepath.Join(tpath, "work"), 0755); err != nil { - glog.Println(err) + log.Println(err) return err } if err = os.MkdirAll(filepath.Join(tpath, "merge"), 0755); err != nil { - glog.Println(err) + log.Println(err) return err } if err = os.MkdirAll(filepath.Join(tpath, "nfs"), 0755); err != nil { - glog.Println(err) + log.Println(err) return err } } @@ -130,7 +132,7 @@ func (p *PreMigrationInTargetMachine) CreateDockerContainer() error { var ( err error ) - args1 := append([]string{"rm"}, p.Cname+"copy") + args1 := append([]string{"rm", "-f"}, p.Cname+"copy") cmd1 := exec.Command("docker", args1...) cmd1.Run() @@ -148,9 +150,9 @@ func (p *PreMigrationInTargetMachine) CreateDockerContainer() error { args = append(args, p.ImageName) cmd := exec.Command("docker", args...) - glog.Printf("create cmd is %v\n", cmd.Args) + log.Printf("create cmd is %v\n", cmd.Args) if err = cmd.Run(); err != nil { - glog.Println(err) + log.Println(err) return err } return nil @@ -162,9 +164,9 @@ func (p *PreMigrationInTargetMachine) StartDockerContainer() error { args = append(args, filepath.Join(RemoteCheckpointDir, p.Id+"copy")) args = append(args, "--checkpoint", DumpAll, name) cmd := exec.Command("docker", args...) - glog.Printf("start docker cmd is %v\n", cmd.Args) + log.Printf("start docker cmd is %v\n", cmd.Args) if err := cmd.Run(); err != nil { - glog.Println(err) + log.Println(err) return err } @@ -181,10 +183,10 @@ func (p *PreMigrationInTargetMachine) CopyUpperDir(imageid string) error { ) src := filepath.Join(remoteUpperDir, imageid, "diff") - //glog.Printf("imageid is %v\n",imageid) + //log.Printf("imageid is %v\n",imageid) _, err = os.Stat(src) if err != nil { - glog.Printf("Remote Don't Has Upperdir %v:", err) + log.Printf("Remote Don't Has Upperdir %v:", err) //return err } @@ -193,12 +195,12 @@ func (p *PreMigrationInTargetMachine) CopyUpperDir(imageid string) error { bs, err := cmd.Output() if err != nil { - glog.Println(err) + log.Println(err) return err } if err = json.Unmarshal(bs, &tmp); err != nil { - glog.Println(err) + log.Println(err) return err } @@ -208,10 +210,10 @@ func (p *PreMigrationInTargetMachine) CopyUpperDir(imageid string) error { src = src + "/" } - glog.Printf("src is is %v,dst is %v\n", src, tmp[0].GraphDriver.Data.UpperDir) + log.Printf("src is is %v,dst is %v\n", src, tmp[0].GraphDriver.Data.UpperDir) if err = CopyDirLocal(src, dst); err != nil { - glog.Println(err) + log.Println(err) return err } @@ -231,9 +233,9 @@ func (p *PreMigrationInTargetMachine) MountNfs() error { args = append(args, filepath.Join(RemoteGetVolume(p.Id, i), "nfs")) cmd := exec.Command("mount", args...) - glog.Printf("mount cmd is %v\n", cmd.Args) + log.Printf("mount cmd is %v\n", cmd.Args) if err = cmd.Run(); err != nil { - glog.Println(err) + log.Println(err) return err } } @@ -266,13 +268,13 @@ func (p *PreMigrationInTargetMachine) PreLazyDir() error { cmd := exec.Command("mount", args...) - glog.Printf("overlay cmd is %v\n", cmd.Args) + log.Printf("overlay cmd is %v\n", cmd.Args) if err = cmd.Run(); err != nil { - glog.Println(err) + log.Println(err) return err } - //glog.Println(cmd.Args) + //log.Println(cmd.Args) } return nil } @@ -284,19 +286,19 @@ func (p *PreMigrationInTargetMachine) StartPreLazyCopy() error { ) for i := 0; i < len(p.Vol); i++ { - glog.Printf("start lazy vol %d\n", i) + log.Printf("start lazy vol %d\n", i) crwdir = filepath.Join(RemoteGetVolume(p.Id, i), "nfs") monidir = filepath.Join(RemoteGetVolume(p.Id, i), "upper") lazydir = filepath.Join(RemoteGetVolume(p.Id, i), "lazy") r := lazycopydir.NewLazyReplicator(crwdir, monidir, lazydir) if err = r.Prelazy(); err != nil { - glog.Println(err) + log.Println(err) return err } lazyreplicator = append(lazyreplicator, r) } - glog.Printf("finish pre lazy copy:%v", time.Now()) + log.Printf("finish pre lazy copy:%v", time.Now()) return nil } @@ -306,7 +308,7 @@ func (p *PreMigrationInTargetMachine) StartLazyCopy() error { ) for _, v := range lazyreplicator { if err = v.Dolazycopy(); err != nil { - glog.Println(err) + log.Println(err) return err } } diff --git a/supervisor/migration/remoteMigration.go b/supervisor/migration/remoteMigration.go index 3efe4ca..9875a9f 100644 --- a/supervisor/migration/remoteMigration.go +++ b/supervisor/migration/remoteMigration.go @@ -6,11 +6,12 @@ import ( "github.com/containerd/containerd/api/grpc/types" "github.com/containerd/containerd/specs" "github.com/pkg/sftp" - "github.com/sirupsen/logrus" + //"github.com/sirupsen/log" netcontext "golang.org/x/net/context" "os" "path/filepath" "time" + "log" ) //const RemoteOverlay="/var/lib/migration/overlay/diff-id.." 远程主机的镜像层文件目录 @@ -38,13 +39,13 @@ type remoteMigration struct { func NewRemoteMigration(ip, id string, port uint32) (*remoteMigration, error) { - logrus.Println("get grpc client") + log.Println("get grpc client") c, err := GetClient(ip, port) if err != nil { return nil, err } - logrus.Println("get sftp client") + log.Println("get sftp client") sc, err := GetSftpClient(LoginUser, LoginPasswd, ip, SftpPort) if err != nil { return nil, err @@ -67,7 +68,7 @@ func NewRemoteMigration(ip, id string, port uint32) (*remoteMigration, error) { func (r *remoteMigration) DoRestore() error { //just log it,do nothing - glog.Println("Do Remote Restore") + log.Println("Do Remote Restore") return nil bpath, err := filepath.Abs(r.Bundle) @@ -87,7 +88,7 @@ func (r *remoteMigration) DoRestore() error { //runc create if _, err = r.clienApi.CreateContainer(netcontext.Background(), req); err != nil { - logrus.Printf("remote restore err:%v\n", err) + log.Printf("remote restore err:%v\n", err) return err } time.Sleep(2 * time.Second) @@ -104,13 +105,13 @@ func (r *remoteMigration) DoRestore() error { func (r *remoteMigration) PreLoadImage(e chan error, image *Image) { - //glog.Println("start precopy image") + //log.Println("start precopy image") //err := image.PreCopyImage(r.sftpClient, r) //if err != nil { - // glog.Println(err) + // log.Println(err) //} var err error = nil - glog.Println("we do nothing,just return to main goroutine") + log.Println("we do nothing,just return to main goroutine") e <- err } @@ -122,7 +123,7 @@ func (r *remoteMigration) SetSpec(l *localMigration) error { } rspec, err := LoadSpec(l.Container) if err != nil { - glog.Println(err) + log.Println(err) return err } //rspec.Root.Path = r.Rootfs @@ -134,34 +135,34 @@ func (r *remoteMigration) SetSpec(l *localMigration) error { if err == os.ErrNotExist { if err = RemoteMkdirAll(rfile, r.sftpClient); err != nil { - glog.Println(err) + log.Println(err) return err } if specf, err := r.sftpClient.Create(rfile); err != nil { - glog.Println(err) + log.Println(err) return err } else { if err = json.NewEncoder(specf).Encode(rspec); err != nil { - glog.Println(err) + log.Println(err) return err } } } else { - glog.Println(err) + log.Println(err) return err } } - //glog.Println("Remote Has Config.json\n") + //log.Println("Remote Has Config.json\n") return nil } //向目的主机发送grpc请求 func (r *remoteMigration) PreRemoteMigration(id, upperid string, args []string) error { - //glog.Printf("upperid is %v\n",upperid) + //log.Printf("upperid is %v\n",upperid) var ( err error Id = id @@ -170,22 +171,22 @@ func (r *remoteMigration) PreRemoteMigration(id, upperid string, args []string) ) vol, err = GetVolume(Id) if err != nil { - glog.Println(err) + log.Println(err) return err } imagename, err = GetImage(Id) if err != nil { - glog.Println(err) + log.Println(err) return err } Cname, err = GetCName(Id) if err != nil { - glog.Println(err) + log.Println(err) return err } srcip, err = GetIp() if err != nil { - glog.Println(err) + log.Println(err) return err } @@ -204,7 +205,7 @@ func (r *remoteMigration) PreRemoteMigration(id, upperid string, args []string) preRequest.Vol = pvol if _, err = r.clienApi.PreMigration(netcontext.Background(), preRequest); err != nil { - glog.Println(err) + log.Println(err) return err } return nil @@ -224,10 +225,10 @@ func (r *remoteMigration) PreRemoteMigration(id, upperid string, args []string) // e.Stderr="/dev/null" // e.BundlePath=r.Bundle // e.StartResponse = make(chan supervisor.StartResponse, 1) -// logrus.Println("restore send task") +// log.Println("restore send task") // s.SendTask(e) // if err := <-e.ErrorCh(); err != nil { -// logrus.Println(err) +// log.Println(err) // return err // } // <-e.StartResponse diff --git a/supervisor/migration/utils_test.go b/supervisor/migration/utils_test.go index a5e24cf..38fc56e 100644 --- a/supervisor/migration/utils_test.go +++ b/supervisor/migration/utils_test.go @@ -178,7 +178,7 @@ func TestCopyUpperDir(t *testing.T) { } //err:=p.CreateDockerContainer() //if err!=nil { - // glog.Println(err) + // log.Println(err) // t.FailNow() // return //} @@ -186,7 +186,7 @@ func TestCopyUpperDir(t *testing.T) { if src[len(src)-1] != '/' { src = src + "/" } - glog.Printf("src is %v\n", src) + log.Printf("src is %v\n", src) return } @@ -203,10 +203,10 @@ func TestGetIp(t *testing.T) { func TestGetCName(t *testing.T) { c, err := GetCName("895fcefdab76") if err != nil { - glog.Println(err) + log.Println(err) t.FailNow() return } - glog.Printf("cname is %v\n", c) + log.Printf("cname is %v\n", c) return } diff --git a/supervisor/preMigration.go b/supervisor/preMigration.go index 7ef5e78..1de8e56 100644 --- a/supervisor/preMigration.go +++ b/supervisor/preMigration.go @@ -2,8 +2,8 @@ package supervisor import ( "github.com/containerd/containerd/supervisor/migration" - "github.com/sirupsen/logrus" "time" + "log" ) const preVolume = "/var/lib/migration/mvolume" @@ -48,9 +48,9 @@ func (s *Supervisor) PreMigration(t *PreMigrationTask) error { Args: t.Args, } - logrus.Printf("start preMigration at %v\n", time.Now()) + log.Printf("start preMigration at %v\n", time.Now()) if err = pre.StartPre(); err != nil { - logrus.Printf("start pre in supervisor error:%v\n", err) + log.Printf("start pre in supervisor error:%v\n", err) return err } @@ -71,7 +71,7 @@ func (s *Supervisor) PreMigration(t *PreMigrationTask) error { // args = append(args, p.ImageName) // cmd := exec.Command("docker", args...) // if err = cmd.Run(); err != nil { -// logrus.Println(err) +// log.Println(err) // return err // } // return nil