xiconet · May 27, 2018 21:30 · May 27, 2018 · May 27, 2018
diff --git a/hitail.go b/hitail.go
@@ -0,0 +1,593 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/alexflint/go-arg"
+	"github.com/cheggaaa/pb"
+	"github.com/nu7hatch/gouuid"
+	"github.com/xiconet/utils"
+	"io"
+	"io/ioutil"
+	"math"
+	"mime/multipart"
+	"net/http"
+	"net/url"
+	"os"
+	ospath "path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"menteslibres.net/gosexy/to"
+	"menteslibres.net/gosexy/yaml"
+)
+
+const (
+	host     = "spaces.hightail.com"
+	cfg_file = "C:/Users/ARC/.config/hightail.yml"
+)
+
+var (
+	auth_url     = fmt.Sprintf("https://api.%s/api/v1/auth", host)
+	base_url     = fmt.Sprintf("https://folders.%s", host)
+	download_url = fmt.Sprintf("https://download.%s/api/v1/download", host)
+	upload_url   = fmt.Sprintf("https://upload.%s/api/v1/upload", host)
+	cookies      = map[string]string{}
+)
+
+type LoginData struct {
+	SessionID string `json:"sessionId"`
+	User      struct {
+		Id             string `json:"id"`
+		Email          string `json "email"`
+		Name           string `json:"name"`
+		Verified       bool   `json:"verified"`
+		IsNative       bool   `json:"isNative"`
+		Status         string `json:"status"`
+		InletType      string `json:"inletType"`
+		AuthType       string `json:"authType"`
+		OrganizationId string `json:"organizationId"`
+		EarlyAccess    bool   `json:"earlyAccess"`
+		UpdatedAt      int    `json:"updatedAt"`
+		CreatedAt      int    `json:"createdAt"`
+		Active         bool   `json:"active"`
+		Native         bool   `json:"native"`
+	} `json:"user"`
+}
+
+type Node struct {
+	CreateDate           string   `json:"createDate"`
+	EffectivePermissions []string `json:"effectivePermissions"`
+	FolderType           string   `json:"folderType"`
+	Id                   string   `json:"id"`
+	IsDeleted            bool     `json:"isDeleted"`
+	IsDirectory          bool     `json:"isDirectory"`
+	ModifyDate           string   `json:"modifyDate"`
+	Name                 string   `json:"name"`
+	OwnerId              string   `json:"ownerId"`
+	PhysicalFileSize     string   `json:"physicalFileSize"`
+	ParentId             string   `json:"parentId"`
+	Revision             string   `json:"revision"`
+}
+
+type Folder struct {
+	Children []Node `json:"children"`
+	Id       string `json:"id"`
+}
+
+type ByName []Node
+
+func (a ByName) Len() int           { return len(a) }
+func (a ByName) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a ByName) Less(i, j int) bool { return a[i].Name < a[j].Name }
+
+type NewDir struct {
+	GeneratedFileId string `json:"generatedFileId"`
+	IsDeleted       bool   `json:"isDeleted"`
+	Revision        string `json:"revision"`
+}
+
+type UpResp struct {
+	ClientCreatedDate string `json:"clientCreatedDate"`
+	ClientUpdatedDate string `json:"clientUpdatedDate"`
+	CreatedDate       string `json:"createdDate"`
+	CreatorId         string `json:"creatorId"`
+	FolderId          string `json:"folderId"`
+	Id                string `json:"id"`
+	LogicalFileId     string `json:"logicalFileId"`
+	Name              string `json:"name"`
+	Region            string `json:"region"`
+	Revision          string `json:"revision"`
+	RevisionStr       string `json:"revisionStr"`
+	Size              string `json:"size"`
+	UfId              string `json:"ufId"`
+	UpdatedDate       string `json:"updatedDate"`
+}
+
+type ReqData struct {
+	Directory   bool   `json:"directory"`
+	Id          string `json:"id,omitempty"`
+	Name        string `json:"name,omitempty"`
+	NewFilename string `json:"newFilename,omitempty"`
+	ParentId    string `json:"parentId,omitempty"`
+}
+
+func userCfg(user string) (email, pwd string) {
+	cfg, err := yaml.Open(cfg_file)
+	if err != nil {
+		panic(err)
+	}
+	if user == "current_user" {
+		user = to.String(cfg.Get("users", "current_user"))
+	}
+	pwd = to.String(cfg.Get(user, "password"))
+	email = to.String(cfg.Get(user, "email"))
+	return
+
+}
+
+func apiReq(method, uri string, data interface{}) (error, string) {
+	var req *http.Request
+	var err error
+	if method == "GET" {
+		req, err = http.NewRequest(method, uri, nil)
+		if err != nil {
+			panic(err)
+		}
+	}
+	if data != nil {
+		js, err := json.Marshal(data)
+		if err != nil {
+			panic(err)
+		}
+		req, err = http.NewRequest(method, uri, strings.NewReader(string(js)))
+		if err != nil {
+			panic(err)
+		}
+		req.Header.Set("Content-Type", "application/json")
+	}
+	if req == nil {
+		req, err = http.NewRequest(method, uri, nil)
+		if err != nil {
+			panic(err)
+		}
+	}
+	if !(strings.Contains(uri, "login")) {
+		for k, v := range cookies {
+			req.Header.Add("Cookie", fmt.Sprintf("%s=%s", k, v))
+		}
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		panic(err)
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		panic(err)
+	}
+	if resp.StatusCode != 200 {
+		fmt.Println("error: bad server status", resp.Status)
+		fmt.Println(string(body))
+	}
+	return nil, string(body)
+}
+
+func login(email, pwd string) {
+	data := map[string]string{"email": email, "password": pwd}
+	uri, _ := url.Parse(auth_url + "/login")
+	err, body := apiReq("POST", uri.String(), data)
+	if err != nil {
+		fmt.Println("login request error:", err)
+		os.Exit(1)
+	}
+	ld := LoginData{}
+	err = json.Unmarshal([]byte(body), &ld)
+	if err != nil {
+		fmt.Println("json error:", err)
+		fmt.Println(body)
+		os.Exit(1)
+	}
+	cookies["sessionId"] = ld.SessionID
+	cookies["userId"] = ld.User.Id
+}
+
+func listFolder(folderId string) (f Folder) {
+	uri, _ := url.Parse(base_url)
+	uri.Path += fmt.Sprintf("/api/v1/hfsEdge/children/%s", folderId)
+	err, body := apiReq("GET", uri.String(), nil)
+	if err != nil {
+		fmt.Println("apiReq error:", err)
+		fmt.Println(body)
+		return
+	}
+	err = json.Unmarshal([]byte(body), &f)
+	if err != nil {
+		fmt.Println("json error:", err)
+		fmt.Println("body:", body)
+		os.Exit(1)
+	}
+	return
+}
+
+func treeList(folderId string, i, depth int) {
+	idt := strings.Repeat(" ", 2*i)
+	node := listFolder(folderId)
+	sort.Sort(ByName(node.Children))
+	for _, c := range node.Children {
+		if !c.IsDirectory {
+			pfs, _ := strconv.Atoi(c.PhysicalFileSize)
+			fsize, _ := utils.NiceBytes(int64(pfs))
+			sfmt := fmt.Sprintf("%%s%%-%ds %%10s\n", 67-2*i)
+			fmt.Printf(sfmt, idt, c.Name, fsize)
+		} else if depth == 0 || i < depth {
+			fmt.Printf("%s%s/\n", idt, c.Name)
+			treeList(c.Id, i+1, depth)
+		}
+	}
+}
+
+func treeSize(folderID string, usedSpace int64) int64 {
+	node := listFolder(folderID)
+	for _, c := range node.Children {
+		if !c.IsDirectory {
+			pfs, _ := strconv.Atoi(c.PhysicalFileSize)
+			usedSpace += int64(pfs)
+		} else {
+			usedSpace = treeSize(c.Id, usedSpace)
+		}
+	}
+	return usedSpace
+}
+
+func pathToID(path, rootID string) (found bool, n Node) {
+	if path == "/" {
+		return true, Node{Id: "0", IsDirectory: true}
+	}
+	for _, p := range strings.Split(path, "/") {
+		found = false
+		children := listFolder(rootID).Children
+		for _, c := range children {
+			if c.Name == p {
+				found = true
+				rootID = c.Id
+				n = c
+				break
+			}
+		}
+		if !found {
+			fmt.Println("error: path not found:", p)
+		}
+	}
+	return
+}
+
+func createFolder(folder_name, parent_id string) (nd NewDir) {
+	uri, _ := url.Parse(base_url + "/api/v1/hfsEdge/create")
+	data := ReqData{Name: folder_name, Directory: true, ParentId: parent_id}
+	err, body := apiReq("POST", uri.String(), data)
+	if err != nil {
+		fmt.Println("error:", err)
+	}
+	err = json.Unmarshal([]byte(body), &nd)
+	if err != nil {
+		fmt.Println(err)
+		fmt.Println(body)
+	}
+	return
+}
+
+func download(n Node, dest string) int64 {
+	if dest == "" {
+		dest = n.Name
+	}
+	uri, _ := url.Parse(download_url)
+	uri.Path += fmt.Sprintf("/link/HIGHTAIL_FILE/%s/%s/%s", n.Id, n.Revision, n.Name)
+	fmt.Println("url:", uri.String())
+	req, err := http.NewRequest("GET", uri.String(), nil)
+	if err != nil {
+		panic(err)
+	}
+	for k, v := range cookies {
+		req.Header.Add("Cookie", fmt.Sprintf("%s=%s", k, v))
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		fmt.Println("request error:", err)
+		fmt.Printf("request: %+v\n", req)
+		os.Exit(1)
+	}
+	if resp.StatusCode != 200 {
+		fmt.Println("error: bad server status:", resp.Status)
+		os.Exit(1)
+	}
+	defer resp.Body.Close()
+	out, err := os.Create(dest)
+	if err != nil {
+		panic(err)
+	}
+	defer out.Close()
+	var srcSize, fl int64
+	cl := resp.Header.Get("Content-Length")
+	if cl != "" {
+		i, err := strconv.Atoi(cl)
+		if err != nil {
+			fmt.Println(err)
+		}
+		srcSize = int64(i)
+	} else {
+		pfs, err := strconv.Atoi(n.PhysicalFileSize)
+		if err != nil {
+			fmt.Println(err)
+		}
+		fl = int64(pfs)
+		srcSize = fl
+	}
+	src := resp.Body
+	bar := pb.New(int(srcSize)).SetUnits(pb.U_BYTES).SetRefreshRate(time.Millisecond * 10)
+	if srcSize >= 1024*1024 {
+		bar.ShowSpeed = true
+	}
+	bar.Start()
+	writer := io.MultiWriter(out, bar)
+	m, err := io.Copy(writer, src)
+	if err != nil {
+		fmt.Println("Error while downloading from:", uri.String(), "-", err)
+		return 0
+	}
+	bar.Finish()
+	return m
+}
+
+func downsync(obj Node, localPath string, tBytes int64) int64 {
+	err := os.MkdirAll(localPath, 0777)
+	if err != nil {
+		fmt.Println("error: could not create new folder", localPath, ":", err)
+		os.Exit(1)
+	}
+	for _, c := range listFolder(obj.Id).Children {
+		if !c.IsDirectory {
+			tBytes += download(c, ospath.Join(localPath, c.Name))
+		} else {
+			tBytes = downsync(c, ospath.Join(localPath, c.Name), tBytes)
+		}
+	}
+	return tBytes
+}
+
+//func GetFileContentType tries to detect a file's mime type
+func GetFileContentType(out *os.File) (string, error) {
+	// Only the first 512 bytes are used to sniff the content type.
+	buffer := make([]byte, 512)
+	_, err := out.Read(buffer)
+	if err != nil {
+		return "", err
+	}
+	// The net/http DectectContentType function returns a valid
+	// "application/octet-stream" if no other match is found.
+	contentType := http.DetectContentType(buffer)
+	return contentType, nil
+}
+
+func makeChunk(fh *os.File, offset int64) []byte {
+	chunksize := int64(5 * 1024 * 1024)
+	p := make([]byte, chunksize)
+	//fmt.Println("offset:", offset)
+	n, _ := fh.ReadAt(p, offset)
+	return p[:n]
+}
+
+func uploadRequest(uri string, params interface{}, chunk []byte) (*http.Request, error) {
+	body := &bytes.Buffer{}
+	ck := bytes.NewBuffer(chunk)
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", "blob")
+	if err != nil {
+		return nil, err
+	}
+	_, err = io.Copy(part, ck)
+	for key, val := range params.(map[string]string) {
+		_ = writer.WriteField(key, val)
+	}
+	err = writer.Close()
+	if err != nil {
+		return nil, err
+	}
+	req, err := http.NewRequest("POST", uri, body)
+	if err != nil {
+		panic(err)
+	}
+	for k, v := range cookies {
+		req.Header.Add("Cookie", fmt.Sprintf("%s=%s", k, v))
+	}
+	req.Header.Add("Content-Type", writer.FormDataContentType())
+	return req, nil
+}
+
+func resumableUpload(fp string, parent Node) (uresp UpResp) {
+	const MCS = int64(5 * 1024 * 1024) // max. chunk size
+	mcs := fmt.Sprintf("%d", MCS)
+	fn := ospath.Base(fp)
+	uri, err := url.Parse(upload_url)
+	uri.Path += fmt.Sprintf("/folder/resumable/%s", parent.Id)
+	if err != nil {
+		panic(err)
+	}
+	fh, err := os.Open(fp)
+	if err != nil {
+		panic(err)
+	}
+	defer fh.Close()
+	stat, err := fh.Stat()
+	if err != nil {
+		panic(err)
+	}
+	fsize := stat.Size()
+	filesize := fmt.Sprintf("%d", fsize)
+	chunks := math.Ceil(float64(fsize) / float64(MCS))
+	tchunks := fmt.Sprintf("%d", chunks)
+	rts := fmt.Sprintf("%d", int64(math.Min(float64(fsize), float64(MCS))))
+	rt, err := GetFileContentType(fh)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	uid, err := uuid.NewV4()
+	if err != nil {
+		panic(err)
+	}
+	rin := uid.String()
+	chunk_number := 1
+	data := map[string]string{
+		"resumableChunkNumber":      "1",
+		"resumableChunkSize":        mcs,
+		"resumableCurrentChunkSize": rts,
+		"resumableTotalSize":        filesize,
+		"resumableType":             rt,
+		"resumableIdentifier":       rin,
+		"resumableFilename":         fn,
+		"resumableRelativePath":     fn,
+		"resumableTotalChunks":      tchunks,
+	}
+	offset := int64(0)
+	for offset < fsize {
+		chunk := makeChunk(fh, offset)
+		data["resumableCurrentChunkSize"] = fmt.Sprintf("%d", len(chunk))
+		req, err := uploadRequest(uri.String(), data, chunk)
+		if err != nil {
+			panic(err)
+		}
+		fmt.Printf("uploading chunk #%s (%d bytes)...\n", data["resumableChunkNumber"], len(chunk))
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			fmt.Println("upload request error:", err)
+			return
+		}
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			fmt.Println("read body error:", err)
+			return
+		}
+		fmt.Println("server status:", resp.Status)
+		if resp.StatusCode != 200 {
+			fmt.Println("error: bad server status:", resp.Status)
+			fmt.Println("reponse body:", string(body))
+			return
+		}
+		err = json.Unmarshal(body, &uresp)
+		if err != nil {
+			fmt.Println(err)
+			fmt.Println("reponse body:", string(body))
+			return
+		}
+		chunk_number += 1
+		data["resumableChunkNumber"] = fmt.Sprintf("%d", chunk_number)
+		offset += int64(len(chunk))
+	}
+	return
+}
+
+func rename(n Node, newname string) string {
+	uri, _ := url.Parse(base_url)
+	uri.Path += "/api/v1/hfsEdge/rename"
+	data := ReqData{Id: n.Id, NewFilename: newname, Directory: n.IsDirectory}
+	err, body := apiReq("POST", uri.String(), data)
+	if err != nil {
+		fmt.Println(err)
+	}
+	return body
+}
+
+func deleteNode(n Node) string {
+	uri, _ := url.Parse(base_url)
+	uri.Path += fmt.Sprintf("/api/v1/hfsEdge/delete")
+	data := ReqData{Id: n.Id, Directory: n.IsDirectory}
+	err, body := apiReq("POST", uri.String(), data)
+	if err != nil {
+		fmt.Println(err)
+	}
+	return body
+}
+
+func main() {
+	var args struct {
+		Path     string `arg:"positional,help:node path"`
+		User     string `arg:"-u,help:account user name"`
+		Tree     bool   `arg:"-t,help:recursive (tree-style) folder list"`
+		Recurse  int    `arg:"-R,help:recursion depth"`
+		Info     bool   `arg:"-i,help:compute total size for the specified folder"`
+		Mkdir    string `arg:"-m,help:create a new folder in the specified parent folder path"`
+		Upload   string `arg:"-p,help:upload a file to the specified parent folder path"`
+		Download bool   `arg:"-d,help:download item(s) under the specified path"`
+		Rename   string `arg:"-r,help:rename item at the specified path"`
+		Remove   bool   `arg:"-x,help:delete item(s) under the specified path"`
+		Verbose  bool   `arg:"-v,help:verbose mode"`
+	}
+	args.Path = "/"
+	args.Recurse = 0
+	args.User = "current_user"
+	arg.MustParse(&args)
+	p := args.Path
+	email, pwd := userCfg(args.User)
+	login(email, pwd)
+	ok, node := pathToID(p, "0")
+	if !ok {
+		os.Exit(1)
+	}
+	fmt.Printf("user: %s; node Id: %s\n\n", email, node.Id)
+	switch {
+	case args.Download:
+		if !node.IsDirectory {
+			download(node, "")
+		} else {
+			dl := downsync(node, node.Name, 0)
+			dls, _ := utils.NiceBytes(dl)
+			fmt.Printf("total downloaded: %s\n", dls)
+		}
+	case args.Info:
+		const quota = int64(2 * 1024 * 1024 * 1024)
+		fmt.Printf("computing tree size for folder %q...\n", args.Path)
+		tsize := treeSize(node.Id, 0)
+		ts, _ := utils.NiceBytes(tsize)
+		if args.Path == "/" {
+			left := quota - tsize
+			l, _ := utils.NiceBytes(left)
+			fmt.Printf("used: %10s\nleft: %10s", ts, l)
+		} else {
+			fmt.Printf("path: %s\ntotal size: %s\n", args.Path, ts)
+		}
+	case args.Tree:
+		treeList(node.Id, 0, args.Recurse)
+	case args.Mkdir != "":
+		if !node.IsDirectory {
+			fmt.Println("error:", node.Name, "is not a folder")
+		} else {
+			newf := createFolder(args.Mkdir, node.Id)
+			fmt.Printf("%+v\n", newf)
+		}
+	case args.Upload != "":
+		resp := resumableUpload(args.Upload, node)
+		fmt.Printf("%+v\n", resp)
+	case args.Rename != "":
+		resp := rename(node, args.Rename)
+		fmt.Println(resp)
+	case args.Remove:
+		resp := deleteNode(node)
+		fmt.Println(resp)
+	default:
+		lf := listFolder(node.Id)
+		sort.Sort(ByName(lf.Children))
+		for _, c := range lf.Children {
+			if c.IsDirectory {
+				fmt.Printf("%s/\n", c.Name)
+			} else {
+				pfs, _ := strconv.Atoi(c.PhysicalFileSize)
+				fsize, _ := utils.NiceBytes(int64(pfs))
+				fmt.Printf("%-67s %10s\n", c.Name, fsize)
+			}
+		}
+	}
+}