package main import ( "bytes" "encoding/json" "fmt" "github.com/alexflint/go-arg" "github.com/cheggaaa/pb" "github.com/nu7hatch/gouuid" "github.com/xiconet/utils" "io" "io/ioutil" "math" "mime/multipart" "net/http" "net/url" "os" ospath "path/filepath" "sort" "strconv" "strings" "time" "menteslibres.net/gosexy/to" "menteslibres.net/gosexy/yaml" ) const ( host = "spaces.hightail.com" cfg_file = "C:/Users/ARC/.config/hightail.yml" ) var ( auth_url = fmt.Sprintf("https://api.%s/api/v1/auth", host) base_url = fmt.Sprintf("https://folders.%s", host) download_url = fmt.Sprintf("https://download.%s/api/v1/download", host) upload_url = fmt.Sprintf("https://upload.%s/api/v1/upload", host) cookies = map[string]string{} ) type LoginData struct { SessionID string `json:"sessionId"` User struct { Id string `json:"id"` Email string `json "email"` Name string `json:"name"` Verified bool `json:"verified"` IsNative bool `json:"isNative"` Status string `json:"status"` InletType string `json:"inletType"` AuthType string `json:"authType"` OrganizationId string `json:"organizationId"` EarlyAccess bool `json:"earlyAccess"` UpdatedAt int `json:"updatedAt"` CreatedAt int `json:"createdAt"` Active bool `json:"active"` Native bool `json:"native"` } `json:"user"` } type Node struct { CreateDate string `json:"createDate"` EffectivePermissions []string `json:"effectivePermissions"` FolderType string `json:"folderType"` Id string `json:"id"` IsDeleted bool `json:"isDeleted"` IsDirectory bool `json:"isDirectory"` ModifyDate string `json:"modifyDate"` Name string `json:"name"` OwnerId string `json:"ownerId"` PhysicalFileSize string `json:"physicalFileSize"` ParentId string `json:"parentId"` Revision string `json:"revision"` } type Folder struct { Children []Node `json:"children"` Id string `json:"id"` } type ByName []Node func (a ByName) Len() int { return len(a) } func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a ByName) Less(i, j int) bool { return a[i].Name < a[j].Name } type NewDir struct { GeneratedFileId string `json:"generatedFileId"` IsDeleted bool `json:"isDeleted"` Revision string `json:"revision"` } type UpResp struct { ClientCreatedDate string `json:"clientCreatedDate"` ClientUpdatedDate string `json:"clientUpdatedDate"` CreatedDate string `json:"createdDate"` CreatorId string `json:"creatorId"` FolderId string `json:"folderId"` Id string `json:"id"` LogicalFileId string `json:"logicalFileId"` Name string `json:"name"` Region string `json:"region"` Revision string `json:"revision"` RevisionStr string `json:"revisionStr"` Size string `json:"size"` UfId string `json:"ufId"` UpdatedDate string `json:"updatedDate"` } type ReqData struct { Directory bool `json:"directory"` Id string `json:"id,omitempty"` Name string `json:"name,omitempty"` NewFilename string `json:"newFilename,omitempty"` ParentId string `json:"parentId,omitempty"` } func userCfg(user string) (email, pwd string) { cfg, err := yaml.Open(cfg_file) if err != nil { panic(err) } if user == "current_user" { user = to.String(cfg.Get("users", "current_user")) } pwd = to.String(cfg.Get(user, "password")) email = to.String(cfg.Get(user, "email")) return } func apiReq(method, uri string, data interface{}) (error, string) { var req *http.Request var err error if method == "GET" { req, err = http.NewRequest(method, uri, nil) if err != nil { panic(err) } } if data != nil { js, err := json.Marshal(data) if err != nil { panic(err) } req, err = http.NewRequest(method, uri, strings.NewReader(string(js))) if err != nil { panic(err) } req.Header.Set("Content-Type", "application/json") } if req == nil { req, err = http.NewRequest(method, uri, nil) if err != nil { panic(err) } } if !(strings.Contains(uri, "login")) { for k, v := range cookies { req.Header.Add("Cookie", fmt.Sprintf("%s=%s", k, v)) } } resp, err := http.DefaultClient.Do(req) if err != nil { panic(err) } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { panic(err) } if resp.StatusCode != 200 { fmt.Println("error: bad server status", resp.Status) fmt.Println(string(body)) } return nil, string(body) } func login(email, pwd string) { data := map[string]string{"email": email, "password": pwd} uri, _ := url.Parse(auth_url + "/login") err, body := apiReq("POST", uri.String(), data) if err != nil { fmt.Println("login request error:", err) os.Exit(1) } ld := LoginData{} err = json.Unmarshal([]byte(body), &ld) if err != nil { fmt.Println("json error:", err) fmt.Println(body) os.Exit(1) } cookies["sessionId"] = ld.SessionID cookies["userId"] = ld.User.Id } func listFolder(folderId string) (f Folder) { uri, _ := url.Parse(base_url) uri.Path += fmt.Sprintf("/api/v1/hfsEdge/children/%s", folderId) err, body := apiReq("GET", uri.String(), nil) if err != nil { fmt.Println("apiReq error:", err) fmt.Println(body) return } err = json.Unmarshal([]byte(body), &f) if err != nil { fmt.Println("json error:", err) fmt.Println("body:", body) os.Exit(1) } return } func treeList(folderId string, i, depth int) { idt := strings.Repeat(" ", 2*i) node := listFolder(folderId) sort.Sort(ByName(node.Children)) for _, c := range node.Children { if !c.IsDirectory { pfs, _ := strconv.Atoi(c.PhysicalFileSize) fsize, _ := utils.NiceBytes(int64(pfs)) sfmt := fmt.Sprintf("%%s%%-%ds %%10s\n", 67-2*i) fmt.Printf(sfmt, idt, c.Name, fsize) } else if depth == 0 || i < depth { fmt.Printf("%s%s/\n", idt, c.Name) treeList(c.Id, i+1, depth) } } } func treeSize(folderID string, usedSpace int64) int64 { node := listFolder(folderID) for _, c := range node.Children { if !c.IsDirectory { pfs, _ := strconv.Atoi(c.PhysicalFileSize) usedSpace += int64(pfs) } else { usedSpace = treeSize(c.Id, usedSpace) } } return usedSpace } func pathToID(path, rootID string) (found bool, n Node) { if path == "/" { return true, Node{Id: "0", IsDirectory: true} } for _, p := range strings.Split(path, "/") { found = false children := listFolder(rootID).Children for _, c := range children { if c.Name == p { found = true rootID = c.Id n = c break } } if !found { fmt.Println("error: path not found:", p) } } return } func createFolder(folder_name, parent_id string) (nd NewDir) { uri, _ := url.Parse(base_url + "/api/v1/hfsEdge/create") data := ReqData{Name: folder_name, Directory: true, ParentId: parent_id} err, body := apiReq("POST", uri.String(), data) if err != nil { fmt.Println("error:", err) } err = json.Unmarshal([]byte(body), &nd) if err != nil { fmt.Println(err) fmt.Println(body) } return } func download(n Node, dest string) int64 { if dest == "" { dest = n.Name } uri, _ := url.Parse(download_url) uri.Path += fmt.Sprintf("/link/HIGHTAIL_FILE/%s/%s/%s", n.Id, n.Revision, n.Name) fmt.Println("url:", uri.String()) req, err := http.NewRequest("GET", uri.String(), nil) if err != nil { panic(err) } for k, v := range cookies { req.Header.Add("Cookie", fmt.Sprintf("%s=%s", k, v)) } resp, err := http.DefaultClient.Do(req) if err != nil { fmt.Println("request error:", err) fmt.Printf("request: %+v\n", req) os.Exit(1) } if resp.StatusCode != 200 { fmt.Println("error: bad server status:", resp.Status) os.Exit(1) } defer resp.Body.Close() out, err := os.Create(dest) if err != nil { panic(err) } defer out.Close() var srcSize, fl int64 cl := resp.Header.Get("Content-Length") if cl != "" { i, err := strconv.Atoi(cl) if err != nil { fmt.Println(err) } srcSize = int64(i) } else { pfs, err := strconv.Atoi(n.PhysicalFileSize) if err != nil { fmt.Println(err) } fl = int64(pfs) srcSize = fl } src := resp.Body bar := pb.New(int(srcSize)).SetUnits(pb.U_BYTES).SetRefreshRate(time.Millisecond * 10) if srcSize >= 1024*1024 { bar.ShowSpeed = true } bar.Start() writer := io.MultiWriter(out, bar) m, err := io.Copy(writer, src) if err != nil { fmt.Println("Error while downloading from:", uri.String(), "-", err) return 0 } bar.Finish() return m } func downsync(obj Node, localPath string, tBytes int64) int64 { err := os.MkdirAll(localPath, 0777) if err != nil { fmt.Println("error: could not create new folder", localPath, ":", err) os.Exit(1) } for _, c := range listFolder(obj.Id).Children { if !c.IsDirectory { tBytes += download(c, ospath.Join(localPath, c.Name)) } else { tBytes = downsync(c, ospath.Join(localPath, c.Name), tBytes) } } return tBytes } //func GetFileContentType tries to detect a file's mime type func GetFileContentType(out *os.File) (string, error) { // Only the first 512 bytes are used to sniff the content type. buffer := make([]byte, 512) _, err := out.Read(buffer) if err != nil { return "", err } // The net/http DectectContentType function returns a valid // "application/octet-stream" if no other match is found. contentType := http.DetectContentType(buffer) return contentType, nil } func makeChunk(fh *os.File, offset int64) []byte { chunksize := int64(5 * 1024 * 1024) p := make([]byte, chunksize) //fmt.Println("offset:", offset) n, _ := fh.ReadAt(p, offset) return p[:n] } func uploadRequest(uri string, params interface{}, chunk []byte) (*http.Request, error) { body := &bytes.Buffer{} ck := bytes.NewBuffer(chunk) writer := multipart.NewWriter(body) part, err := writer.CreateFormFile("file", "blob") if err != nil { return nil, err } _, err = io.Copy(part, ck) for key, val := range params.(map[string]string) { _ = writer.WriteField(key, val) } err = writer.Close() if err != nil { return nil, err } req, err := http.NewRequest("POST", uri, body) if err != nil { panic(err) } for k, v := range cookies { req.Header.Add("Cookie", fmt.Sprintf("%s=%s", k, v)) } req.Header.Add("Content-Type", writer.FormDataContentType()) return req, nil } func resumableUpload(fp string, parent Node) (uresp UpResp) { const MCS = int64(5 * 1024 * 1024) // max. chunk size mcs := fmt.Sprintf("%d", MCS) fn := ospath.Base(fp) uri, err := url.Parse(upload_url) uri.Path += fmt.Sprintf("/folder/resumable/%s", parent.Id) if err != nil { panic(err) } fh, err := os.Open(fp) if err != nil { panic(err) } defer fh.Close() stat, err := fh.Stat() if err != nil { panic(err) } fsize := stat.Size() filesize := fmt.Sprintf("%d", fsize) chunks := math.Ceil(float64(fsize) / float64(MCS)) tchunks := fmt.Sprintf("%d", chunks) rts := fmt.Sprintf("%d", int64(math.Min(float64(fsize), float64(MCS)))) rt, err := GetFileContentType(fh) if err != nil { fmt.Println(err) os.Exit(1) } uid, err := uuid.NewV4() if err != nil { panic(err) } rin := uid.String() chunk_number := 1 data := map[string]string{ "resumableChunkNumber": "1", "resumableChunkSize": mcs, "resumableCurrentChunkSize": rts, "resumableTotalSize": filesize, "resumableType": rt, "resumableIdentifier": rin, "resumableFilename": fn, "resumableRelativePath": fn, "resumableTotalChunks": tchunks, } offset := int64(0) for offset < fsize { chunk := makeChunk(fh, offset) data["resumableCurrentChunkSize"] = fmt.Sprintf("%d", len(chunk)) req, err := uploadRequest(uri.String(), data, chunk) if err != nil { panic(err) } fmt.Printf("uploading chunk #%s (%d bytes)...\n", data["resumableChunkNumber"], len(chunk)) resp, err := http.DefaultClient.Do(req) if err != nil { fmt.Println("upload request error:", err) return } body, err := ioutil.ReadAll(resp.Body) if err != nil { fmt.Println("read body error:", err) return } fmt.Println("server status:", resp.Status) if resp.StatusCode != 200 { fmt.Println("error: bad server status:", resp.Status) fmt.Println("reponse body:", string(body)) return } err = json.Unmarshal(body, &uresp) if err != nil { fmt.Println(err) fmt.Println("reponse body:", string(body)) return } chunk_number += 1 data["resumableChunkNumber"] = fmt.Sprintf("%d", chunk_number) offset += int64(len(chunk)) } return } func rename(n Node, newname string) string { uri, _ := url.Parse(base_url) uri.Path += "/api/v1/hfsEdge/rename" data := ReqData{Id: n.Id, NewFilename: newname, Directory: n.IsDirectory} err, body := apiReq("POST", uri.String(), data) if err != nil { fmt.Println(err) } return body } func deleteNode(n Node) string { uri, _ := url.Parse(base_url) uri.Path += fmt.Sprintf("/api/v1/hfsEdge/delete") data := ReqData{Id: n.Id, Directory: n.IsDirectory} err, body := apiReq("POST", uri.String(), data) if err != nil { fmt.Println(err) } return body } func main() { var args struct { Path string `arg:"positional,help:node path"` User string `arg:"-u,help:account user name"` Tree bool `arg:"-t,help:recursive (tree-style) folder list"` Recurse int `arg:"-R,help:recursion depth"` Info bool `arg:"-i,help:compute total size for the specified folder"` Mkdir string `arg:"-m,help:create a new folder in the specified parent folder path"` Upload string `arg:"-p,help:upload a file to the specified parent folder path"` Download bool `arg:"-d,help:download item(s) under the specified path"` Rename string `arg:"-r,help:rename item at the specified path"` Remove bool `arg:"-x,help:delete item(s) under the specified path"` Verbose bool `arg:"-v,help:verbose mode"` } args.Path = "/" args.Recurse = 0 args.User = "current_user" arg.MustParse(&args) p := args.Path email, pwd := userCfg(args.User) login(email, pwd) ok, node := pathToID(p, "0") if !ok { os.Exit(1) } fmt.Printf("user: %s; node Id: %s\n\n", email, node.Id) switch { case args.Download: if !node.IsDirectory { download(node, "") } else { dl := downsync(node, node.Name, 0) dls, _ := utils.NiceBytes(dl) fmt.Printf("total downloaded: %s\n", dls) } case args.Info: const quota = int64(2 * 1024 * 1024 * 1024) fmt.Printf("computing tree size for folder %q...\n", args.Path) tsize := treeSize(node.Id, 0) ts, _ := utils.NiceBytes(tsize) if args.Path == "/" { left := quota - tsize l, _ := utils.NiceBytes(left) fmt.Printf("used: %10s\nleft: %10s", ts, l) } else { fmt.Printf("path: %s\ntotal size: %s\n", args.Path, ts) } case args.Tree: treeList(node.Id, 0, args.Recurse) case args.Mkdir != "": if !node.IsDirectory { fmt.Println("error:", node.Name, "is not a folder") } else { newf := createFolder(args.Mkdir, node.Id) fmt.Printf("%+v\n", newf) } case args.Upload != "": resp := resumableUpload(args.Upload, node) fmt.Printf("%+v\n", resp) case args.Rename != "": resp := rename(node, args.Rename) fmt.Println(resp) case args.Remove: resp := deleteNode(node) fmt.Println(resp) default: lf := listFolder(node.Id) sort.Sort(ByName(lf.Children)) for _, c := range lf.Children { if c.IsDirectory { fmt.Printf("%s/\n", c.Name) } else { pfs, _ := strconv.Atoi(c.PhysicalFileSize) fsize, _ := utils.NiceBytes(int64(pfs)) fmt.Printf("%-67s %10s\n", c.Name, fsize) } } } }