Merge pull request #57 from spf13/short-test-sleep
Speed up test on unix systems
diff --git a/README.md b/README.md
index 3a624ea..83310e5 100644
--- a/README.md
+++ b/README.md
@@ -263,12 +263,17 @@
systems with ease. Plans are to add a radix tree memory stored file
system using InMemoryFile.
+## SftpFs
+
+Afero has experimental support for secure file transfer protocol (sftp). Which can
+be used to perform file operations over a encrypted channel.
+
## Desired/possible backends
The following is a short list of possible backends we hope someone will
implement:
-* SSH/SCP
+* SSH
* ZIP
* TAR
* S3
@@ -361,6 +366,7 @@
* [spf13](https://github.com/spf13)
* [jaqx0r](https://github.com/jaqx0r)
* [mbertschler](https://github.com/mbertschler)
+* [xor-gate](https://github.com/xor-gate)
## License
diff --git a/sftp.go b/sftp.go
new file mode 100644
index 0000000..a095a54
--- /dev/null
+++ b/sftp.go
@@ -0,0 +1,128 @@
+// Copyright © 2015 Jerry Jacobs <jerry.jacobs@xor-gate.org>.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package afero
+
+import (
+ "os"
+ "time"
+
+ "github.com/spf13/afero/sftp"
+
+ "github.com/pkg/sftp"
+)
+
+// SftpFs is a Fs implementation that uses functions provided by the sftp package.
+//
+// For details in any method, check the documentation of the sftp package
+// (github.com/pkg/sftp).
+type SftpFs struct{
+ SftpClient *sftp.Client
+}
+
+func (s SftpFs) Name() string { return "SftpFs" }
+
+func (s SftpFs) Create(name string) (File, error) {
+ f, err := sftpfs.FileCreate(s.SftpClient, name)
+ return f, err
+}
+
+func (s SftpFs) Mkdir(name string, perm os.FileMode) error {
+ err := s.SftpClient.Mkdir(name)
+ if err != nil {
+ return err
+ }
+ return s.SftpClient.Chmod(name, perm)
+}
+
+func (s SftpFs) MkdirAll(path string, perm os.FileMode) error {
+ // Fast path: if we can tell whether path is a directory or file, stop with success or error.
+ dir, err := s.Stat(path)
+ if err == nil {
+ if dir.IsDir() {
+ return nil
+ }
+ return err
+ }
+
+ // Slow path: make sure parent exists and then call Mkdir for path.
+ i := len(path)
+ for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
+ i--
+ }
+
+ j := i
+ for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
+ j--
+ }
+
+ if j > 1 {
+ // Create parent
+ err = s.MkdirAll(path[0:j-1], perm)
+ if err != nil {
+ return err
+ }
+ }
+
+ // Parent now exists; invoke Mkdir and use its result.
+ err = s.Mkdir(path, perm)
+ if err != nil {
+ // Handle arguments like "foo/." by
+ // double-checking that directory doesn't exist.
+ dir, err1 := s.Lstat(path)
+ if err1 == nil && dir.IsDir() {
+ return nil
+ }
+ return err
+ }
+ return nil
+}
+
+func (s SftpFs) Open(name string) (File, error) {
+ f, err := sftpfs.FileOpen(s.SftpClient, name)
+ return f, err
+}
+
+func (s SftpFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
+ return nil,nil
+}
+
+func (s SftpFs) Remove(name string) error {
+ return s.SftpClient.Remove(name)
+}
+
+func (s SftpFs) RemoveAll(path string) error {
+ // TODO have a look at os.RemoveAll
+ // https://github.com/golang/go/blob/master/src/os/path.go#L66
+ return nil
+}
+
+func (s SftpFs) Rename(oldname, newname string) error {
+ return s.SftpClient.Rename(oldname, newname)
+}
+
+func (s SftpFs) Stat(name string) (os.FileInfo, error) {
+ return s.SftpClient.Stat(name)
+}
+
+func (s SftpFs) Lstat(p string) (os.FileInfo, error) {
+ return s.SftpClient.Lstat(p)
+}
+
+func (s SftpFs) Chmod(name string, mode os.FileMode) error {
+ return s.SftpClient.Chmod(name, mode)
+}
+
+func (s SftpFs) Chtimes(name string, atime time.Time, mtime time.Time) error {
+ return s.SftpClient.Chtimes(name, atime, mtime)
+}
diff --git a/sftp/file.go b/sftp/file.go
new file mode 100644
index 0000000..bab4abc
--- /dev/null
+++ b/sftp/file.go
@@ -0,0 +1,95 @@
+// Copyright © 2015 Jerry Jacobs <jerry.jacobs@xor-gate.org>.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sftpfs
+
+import (
+ "os"
+ "github.com/pkg/sftp"
+)
+
+type File struct {
+ fd *sftp.File
+}
+
+func FileOpen(s *sftp.Client, name string) (*File, error) {
+ fd, err := s.Open(name)
+ if err != nil {
+ return &File{}, err
+ }
+ return &File{fd: fd}, nil
+}
+
+func FileCreate(s *sftp.Client, name string) (*File, error) {
+ fd, err := s.Create(name)
+ if err != nil {
+ return &File{}, err
+ }
+ return &File{fd: fd}, nil
+}
+
+func (f *File) Close() error {
+ return f.fd.Close()
+}
+
+func (f *File) Name() string {
+ return f.fd.Name()
+}
+
+func (f *File) Stat() (os.FileInfo, error) {
+ return f.fd.Stat()
+}
+
+func (f *File) Sync() error {
+ return nil
+}
+
+func (f *File) Truncate(size int64) error {
+ return f.fd.Truncate(size)
+}
+
+func (f *File) Read(b []byte) (n int, err error) {
+ return f.fd.Read(b)
+}
+
+// TODO
+func (f *File) ReadAt(b []byte, off int64) (n int, err error) {
+ return 0,nil
+}
+
+// TODO
+func (f *File) Readdir(count int) (res []os.FileInfo, err error) {
+ return nil,nil
+}
+
+// TODO
+func (f *File) Readdirnames(n int) (names []string, err error) {
+ return nil,nil
+}
+
+func (f *File) Seek(offset int64, whence int) (int64, error) {
+ return f.fd.Seek(offset, whence)
+}
+
+func (f *File) Write(b []byte) (n int, err error) {
+ return f.fd.Write(b)
+}
+
+// TODO
+func (f *File) WriteAt(b []byte, off int64) (n int, err error) {
+ return 0,nil
+}
+
+func (f *File) WriteString(s string) (ret int, err error) {
+ return f.fd.Write([]byte(s))
+}
diff --git a/sftp_test.go b/sftp_test.go
new file mode 100644
index 0000000..bb00535
--- /dev/null
+++ b/sftp_test.go
@@ -0,0 +1,286 @@
+// Copyright © 2015 Jerry Jacobs <jerry.jacobs@xor-gate.org>.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package afero
+
+import (
+ "testing"
+ "os"
+ "log"
+ "fmt"
+ "net"
+ "flag"
+ "time"
+ "io/ioutil"
+ "crypto/rsa"
+ _rand "crypto/rand"
+ "encoding/pem"
+ "crypto/x509"
+
+ "golang.org/x/crypto/ssh"
+ "github.com/pkg/sftp"
+)
+
+type SftpFsContext struct {
+ sshc *ssh.Client
+ sshcfg *ssh.ClientConfig
+ sftpc *sftp.Client
+}
+
+// TODO we only connect with hardcoded user+pass for now
+// it should be possible to use $HOME/.ssh/id_rsa to login into the stub sftp server
+func SftpConnect(user, password, host string) (*SftpFsContext, error) {
+/*
+ pemBytes, err := ioutil.ReadFile(os.Getenv("HOME") + "/.ssh/id_rsa")
+ if err != nil {
+ return nil,err
+ }
+
+ signer, err := ssh.ParsePrivateKey(pemBytes)
+ if err != nil {
+ return nil,err
+ }
+
+ sshcfg := &ssh.ClientConfig{
+ User: user,
+ Auth: []ssh.AuthMethod{
+ ssh.Password(password),
+ ssh.PublicKeys(signer),
+ },
+ }
+*/
+
+ sshcfg := &ssh.ClientConfig{
+ User: user,
+ Auth: []ssh.AuthMethod{
+ ssh.Password(password),
+ },
+ }
+
+ sshc, err := ssh.Dial("tcp", host, sshcfg)
+ if err != nil {
+ return nil,err
+ }
+
+ sftpc, err := sftp.NewClient(sshc)
+ if err != nil {
+ return nil,err
+ }
+
+ ctx := &SftpFsContext{
+ sshc: sshc,
+ sshcfg: sshcfg,
+ sftpc: sftpc,
+ }
+
+ return ctx,nil
+}
+
+func (ctx *SftpFsContext) Disconnect() error {
+ ctx.sftpc.Close()
+ ctx.sshc.Close()
+ return nil
+}
+
+// TODO for such a weird reason rootpath is "." when writing "file1" with afero sftp backend
+func RunSftpServer(rootpath string) {
+ var (
+ readOnly bool
+ debugLevelStr string
+ debugLevel int
+ debugStderr bool
+ rootDir string
+ )
+
+ flag.BoolVar(&readOnly, "R", false, "read-only server")
+ flag.BoolVar(&debugStderr, "e", true, "debug to stderr")
+ flag.StringVar(&debugLevelStr, "l", "none", "debug level")
+ flag.StringVar(&rootDir, "root", rootpath, "root directory")
+ flag.Parse()
+
+ debugStream := ioutil.Discard
+ if debugStderr {
+ debugStream = os.Stderr
+ debugLevel = 1
+ }
+
+ // An SSH server is represented by a ServerConfig, which holds
+ // certificate details and handles authentication of ServerConns.
+ config := &ssh.ServerConfig{
+ PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
+ // Should use constant-time compare (or better, salt+hash) in
+ // a production setting.
+ fmt.Fprintf(debugStream, "Login: %s\n", c.User())
+ if c.User() == "test" && string(pass) == "test" {
+ return nil, nil
+ }
+ return nil, fmt.Errorf("password rejected for %q", c.User())
+ },
+ }
+
+ privateBytes, err := ioutil.ReadFile("./test/id_rsa")
+ if err != nil {
+ log.Fatal("Failed to load private key", err)
+ }
+
+ private, err := ssh.ParsePrivateKey(privateBytes)
+ if err != nil {
+ log.Fatal("Failed to parse private key", err)
+ }
+
+ config.AddHostKey(private)
+
+ // Once a ServerConfig has been configured, connections can be
+ // accepted.
+ listener, err := net.Listen("tcp", "0.0.0.0:2022")
+ if err != nil {
+ log.Fatal("failed to listen for connection", err)
+ }
+ fmt.Printf("Listening on %v\n", listener.Addr())
+
+ nConn, err := listener.Accept()
+ if err != nil {
+ log.Fatal("failed to accept incoming connection", err)
+ }
+
+ // Before use, a handshake must be performed on the incoming
+ // net.Conn.
+ _, chans, reqs, err := ssh.NewServerConn(nConn, config)
+ if err != nil {
+ log.Fatal("failed to handshake", err)
+ }
+ fmt.Fprintf(debugStream, "SSH server established\n")
+
+ // The incoming Request channel must be serviced.
+ go ssh.DiscardRequests(reqs)
+
+ // Service the incoming Channel channel.
+ for newChannel := range chans {
+ // Channels have a type, depending on the application level
+ // protocol intended. In the case of an SFTP session, this is "subsystem"
+ // with a payload string of "<length=4>sftp"
+ fmt.Fprintf(debugStream, "Incoming channel: %s\n", newChannel.ChannelType())
+ if newChannel.ChannelType() != "session" {
+ newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
+ fmt.Fprintf(debugStream, "Unknown channel type: %s\n", newChannel.ChannelType())
+ continue
+ }
+ channel, requests, err := newChannel.Accept()
+ if err != nil {
+ log.Fatal("could not accept channel.", err)
+ }
+ fmt.Fprintf(debugStream, "Channel accepted\n")
+
+ // Sessions have out-of-band requests such as "shell",
+ // "pty-req" and "env". Here we handle only the
+ // "subsystem" request.
+ go func(in <-chan *ssh.Request) {
+ for req := range in {
+ fmt.Fprintf(debugStream, "Request: %v\n", req.Type)
+ ok := false
+ switch req.Type {
+ case "subsystem":
+ fmt.Fprintf(debugStream, "Subsystem: %s\n", req.Payload[4:])
+ if string(req.Payload[4:]) == "sftp" {
+ ok = true
+ }
+ }
+ fmt.Fprintf(debugStream, " - accepted: %v\n", ok)
+ req.Reply(ok, nil)
+ }
+ }(requests)
+
+ server, err := sftp.NewServer(channel, channel, debugStream, debugLevel, readOnly, rootpath)
+ if err != nil {
+ log.Fatal(err)
+ }
+ if err := server.Serve(); err != nil {
+ log.Fatal("sftp server completed with error:", err)
+ }
+ }
+}
+
+// MakeSSHKeyPair make a pair of public and private keys for SSH access.
+// Public key is encoded in the format for inclusion in an OpenSSH authorized_keys file.
+// Private Key generated is PEM encoded
+func MakeSSHKeyPair(bits int, pubKeyPath, privateKeyPath string) error {
+ privateKey, err := rsa.GenerateKey(_rand.Reader, bits)
+ if err != nil {
+ return err
+ }
+
+ // generate and write private key as PEM
+ privateKeyFile, err := os.Create(privateKeyPath)
+ defer privateKeyFile.Close()
+ if err != nil {
+ return err
+ }
+
+ privateKeyPEM := &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)}
+ if err := pem.Encode(privateKeyFile, privateKeyPEM); err != nil {
+ return err
+ }
+
+ // generate and write public key
+ pub, err := ssh.NewPublicKey(&privateKey.PublicKey)
+ if err != nil {
+ return err
+ }
+
+ return ioutil.WriteFile(pubKeyPath, ssh.MarshalAuthorizedKey(pub), 0655)
+}
+
+func TestSftpCreate(t *testing.T) {
+ os.Mkdir("./test", 0777)
+ MakeSSHKeyPair(1024, "./test/id_rsa.pub", "./test/id_rsa")
+
+ go RunSftpServer("./test/")
+ time.Sleep(5 * time.Second)
+
+ ctx, err := SftpConnect("test", "test", "localhost:2022")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer ctx.Disconnect()
+
+ var AppFs Fs = SftpFs{
+ SftpClient: ctx.sftpc,
+ }
+
+ AppFs.MkdirAll("test/dir1/dir2/dir3", os.FileMode(0777))
+ AppFs.Mkdir("test/foo", os.FileMode(0000))
+ AppFs.Chmod("test/foo", os.FileMode(0700))
+ AppFs.Mkdir("test/bar", os.FileMode(0777))
+
+ file, err := AppFs.Create("file1")
+ if err != nil {
+ t.Error(err)
+ }
+ defer file.Close()
+
+ file.Write([]byte("hello\t"))
+ file.WriteString("world!\n")
+
+ f1, err := AppFs.Open("file1")
+ if err != nil {
+ log.Fatalf("open: %v", err)
+ }
+ defer f1.Close()
+
+ b := make([]byte, 100)
+
+ _, err = f1.Read(b)
+ fmt.Println(string(b))
+
+ // TODO check here if "hello\tworld\n" is in buffer b
+}
