applications/linx-server
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

use abstracted storage for flexibility

Browse Source 
I moved the storage functionality into the StorageBackend interface,
which is currently only implemented by LocalfsBackend.
This commit is contained in:
mutantmonkey 2016-06-06 23:37:42 -07:00
parent 47670af185
commit fcd18eceec
10 changed files with 154 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
backends/backends.go Normal file
View File

@ -0,0 +1,23 @@
package backends
import (
"io"
"net/http"
)
type ReadSeekCloser interface {
io.Reader
io.Closer
io.Seeker
io.ReaderAt
}
type StorageBackend interface {
Delete(key string) error
Exists(key string) (bool, error)
Get(key string) ([]byte, error)
Put(key string, r io.Reader) (int64, error)
Open(key string) (ReadSeekCloser, error)
ServeFile(key string, w http.ResponseWriter, r *http.Request)
Size(key string) (int64, error)
}

70
backends/localfs/localfs.go Normal file
View File

@ -0,0 +1,70 @@
package localfs
import (
"errors"
"io"
"io/ioutil"
"net/http"
"os"
"path"
"github.com/andreimarcu/linx-server/backends"
)
type LocalfsBackend struct {
basePath string
}
func (b LocalfsBackend) Delete(key string) error {
return os.Remove(path.Join(b.basePath, key))
}
func (b LocalfsBackend) Exists(key string) (bool, error) {
_, err := os.Stat(path.Join(b.basePath, key))
return err == nil, err
}
func (b LocalfsBackend) Get(key string) ([]byte, error) {
return ioutil.ReadFile(path.Join(b.basePath, key))
}
func (b LocalfsBackend) Put(key string, r io.Reader) (int64, error) {
dst, err := os.Create(path.Join(b.basePath, key))
if err != nil {
return 0, err
}
defer dst.Close()
bytes, err := io.Copy(dst, r)
if bytes == 0 {
b.Delete(key)
return bytes, errors.New("Empty file")
} else if err != nil {
b.Delete(key)
return bytes, err
}
return bytes, err
}
func (b LocalfsBackend) Open(key string) (backends.ReadSeekCloser, error) {
return os.Open(path.Join(b.basePath, key))
}
func (b LocalfsBackend) ServeFile(key string, w http.ResponseWriter, r *http.Request) {
filePath := path.Join(b.basePath, key)
http.ServeFile(w, r, filePath)
}
func (b LocalfsBackend) Size(key string) (int64, error) {
fileInfo, err := os.Stat(path.Join(b.basePath, key))
if err != nil {
return 0, err
}
return fileInfo.Size(), nil
}
func NewLocalfsBackend(basePath string) LocalfsBackend {
return LocalfsBackend{basePath: basePath}
}

9
delete.go
View File

@ -4,7 +4,6 @@ import (
"fmt"
"net/http"
"os"
"path"
"github.com/zenazn/goji/web"
)
@ -13,11 +12,9 @@ func deleteHandler(c web.C, w http.ResponseWriter, r *http.Request) {
requestKey := r.Header.Get("Linx-Delete-Key")
filename := c.URLParams["name"]
filePath := path.Join(Config.filesDir, filename)
metaPath := path.Join(Config.metaDir, filename)
// Ensure requested file actually exists
if _, readErr := os.Stat(filePath); os.IsNotExist(readErr) {
if _, readErr := fileBackend.Exists(filename); os.IsNotExist(readErr) {
notFoundHandler(c, w, r) // 404 - file doesn't exist
return
}
@ -30,8 +27,8 @@ func deleteHandler(c web.C, w http.ResponseWriter, r *http.Request) {
}
if metadata.DeleteKey == requestKey {
fileDelErr := os.Remove(filePath)
metaDelErr := os.Remove(metaPath)
fileDelErr := fileBackend.Delete(filename)
metaDelErr := metaBackend.Delete(filename)
if (fileDelErr != nil) || (metaDelErr != nil) {
oopsHandler(c, w, r, RespPLAIN, "Could not delete")

12
display.go
View File

@ -2,10 +2,7 @@ package main
import (
"encoding/json"
"io/ioutil"
"net/http"
"os"
"path"
"path/filepath"
"strconv"
"strings"
@ -22,7 +19,6 @@ const maxDisplayFileSizeBytes = 1024 * 512
func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
fileName := c.URLParams["name"]
filePath := path.Join(Config.filesDir, fileName)
err := checkFile(fileName)
if err == NotFoundErr {
@ -43,7 +39,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
extra := make(map[string]string)
lines := []string{}
file, _ := os.Open(filePath)
file, _ := fileBackend.Open(fileName)
defer file.Close()
header := make([]byte, 512)
@ -79,7 +75,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
} else if extension == "story" {
if metadata.Size < maxDisplayFileSizeBytes {
bytes, err := ioutil.ReadFile(filePath)
bytes, err := fileBackend.Get(fileName)
if err == nil {
extra["contents"] = string(bytes)
lines = strings.Split(extra["contents"], "\n")
@ -89,7 +85,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
} else if extension == "md" {
if metadata.Size < maxDisplayFileSizeBytes {
bytes, err := ioutil.ReadFile(filePath)
bytes, err := fileBackend.Get(fileName)
if err == nil {
unsafe := blackfriday.MarkdownCommon(bytes)
html := bluemonday.UGCPolicy().SanitizeBytes(unsafe)
@ -101,7 +97,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
} else if strings.HasPrefix(metadata.Mimetype, "text/") || supportedBinExtension(extension) {
if metadata.Size < maxDisplayFileSizeBytes {
bytes, err := ioutil.ReadFile(filePath)
bytes, err := fileBackend.Get(fileName)
if err == nil {
extra["extension"] = extension
extra["lang_hl"], extra["lang_ace"] = extensionToHlAndAceLangs(extension)

13
fileserve.go
View File

@ -3,8 +3,6 @@ package main
import (
"net/http"
"net/url"
"os"
"path"
"strings"
"github.com/zenazn/goji/web"
@ -12,7 +10,6 @@ import (
func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) {
fileName := c.URLParams["name"]
filePath := path.Join(Config.filesDir, fileName)
err := checkFile(fileName)
if err == NotFoundErr {
@ -35,7 +32,7 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Security-Policy", Config.fileContentSecurityPolicy)
http.ServeFile(w, r, filePath)
fileBackend.ServeFile(fileName, w, r)
}
func staticHandler(c web.C, w http.ResponseWriter, r *http.Request) {
@ -63,9 +60,7 @@ func staticHandler(c web.C, w http.ResponseWriter, r *http.Request) {
}
func checkFile(filename string) error {
filePath := path.Join(Config.filesDir, filename)
_, err := os.Stat(filePath)
_, err := fileBackend.Exists(filename)
if err != nil {
return NotFoundErr
}
@ -76,8 +71,8 @@ func checkFile(filename string) error {
}
if expired {
os.Remove(path.Join(Config.filesDir, filename))
os.Remove(path.Join(Config.metaDir, filename))
fileBackend.Delete(filename)
metaBackend.Delete(filename)
return NotFoundErr
}

27
meta.go
View File

@ -3,6 +3,7 @@ package main
import (
"archive/tar"
"archive/zip"
"bytes"
"compress/bzip2"
"compress/gzip"
"crypto/sha256"
@ -10,9 +11,6 @@ import (
"encoding/json"
"errors"
"io"
"io/ioutil"
"os"
"path"
"sort"
"time"
"unicode"
@ -43,14 +41,17 @@ var NotFoundErr = errors.New("File not found.")
var BadMetadata = errors.New("Corrupted metadata.")
func generateMetadata(fName string, exp time.Time, delKey string) (m Metadata, err error) {
file, err := os.Open(path.Join(Config.filesDir, fName))
fileInfo, err := os.Stat(path.Join(Config.filesDir, fName))
file, err := fileBackend.Open(fName)
if err != nil {
return
}
defer file.Close()
m.Size = fileInfo.Size()
m.Size, err = fileBackend.Size(fName)
if err != nil {
return
}
m.Expiry = exp
if delKey == "" {
@ -138,12 +139,6 @@ func generateMetadata(fName string, exp time.Time, delKey string) (m Metadata, e
}
func metadataWrite(filename string, metadata *Metadata) error {
file, err := os.Create(path.Join(Config.metaDir, filename))
if err != nil {
return err
}
defer file.Close()
mjson := MetadataJSON{}
mjson.DeleteKey = metadata.DeleteKey
mjson.Mimetype = metadata.Mimetype
@ -157,8 +152,7 @@ func metadataWrite(filename string, metadata *Metadata) error {
return err
}
_, err = file.Write(byt)
if err != nil {
if _, err := metaBackend.Put(filename, bytes.NewBuffer(byt)); err != nil {
return err
}
@ -166,7 +160,7 @@ func metadataWrite(filename string, metadata *Metadata) error {
}
func metadataRead(filename string) (metadata Metadata, err error) {
b, err := ioutil.ReadFile(path.Join(Config.metaDir, filename))
b, err := metaBackend.Get(filename)
if err != nil {
// Metadata does not exist, generate one
newMData, err := generateMetadata(filename, neverExpire, "")
@ -174,7 +168,8 @@ func metadataRead(filename string) (metadata Metadata, err error) {
return metadata, err
}
metadataWrite(filename, &newMData)
b, err = ioutil.ReadFile(path.Join(Config.metaDir, filename))
b, err = metaBackend.Get(filename)
if err != nil {
return metadata, BadMetadata
}

7
server.go
View File

@ -14,6 +14,8 @@ import (
"time"
"github.com/GeertJohan/go.rice"
"github.com/andreimarcu/linx-server/backends"
"github.com/andreimarcu/linx-server/backends/localfs"
"github.com/flosch/pongo2"
"github.com/vharitonsky/iniflags"
"github.com/zenazn/goji/graceful"
@ -61,6 +63,8 @@ var staticBox *rice.Box
var timeStarted time.Time
var timeStartedStr string
var remoteAuthKeys []string
var metaBackend backends.StorageBackend
var fileBackend backends.StorageBackend
func setup() *web.Mux {
mux := web.New()
@ -118,6 +122,9 @@ func setup() *web.Mux {
Config.sitePath = "/"
}
metaBackend = localfs.NewLocalfsBackend(Config.metaDir)
fileBackend = localfs.NewLocalfsBackend(Config.filesDir)
// Template setup
p2l, err := NewPongo2TemplatesLoader()
if err != nil {

21
torrent.go
View File

@ -6,8 +6,6 @@ import (
"fmt"
"io"
"net/http"
"os"
"path"
"time"
"github.com/zeebo/bencode"
@ -37,7 +35,7 @@ func hashPiece(piece []byte) []byte {
return h.Sum(nil)
}
func createTorrent(fileName string, filePath string, r *http.Request) ([]byte, error) {
func createTorrent(fileName string, f io.ReadCloser, r *http.Request) ([]byte, error) {
chunk := make([]byte, TORRENT_PIECE_LENGTH)
torrent := Torrent{
@ -49,11 +47,6 @@ func createTorrent(fileName string, filePath string, r *http.Request) ([]byte, e
UrlList: []string{fmt.Sprintf("%sselif/%s", getSiteURL(r), fileName)},
}
f, err := os.Open(filePath)
if err != nil {
return []byte{}, err
}
for {
n, err := f.Read(chunk)
if err == io.EOF {
@ -66,8 +59,6 @@ func createTorrent(fileName string, filePath string, r *http.Request) ([]byte, e
torrent.Info.Pieces += string(hashPiece(chunk[:n]))
}
f.Close()
data, err := bencode.EncodeBytes(&torrent)
if err != nil {
return []byte{}, err
@ -78,7 +69,6 @@ func createTorrent(fileName string, filePath string, r *http.Request) ([]byte, e
func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) {
fileName := c.URLParams["name"]
filePath := path.Join(Config.filesDir, fileName)
err := checkFile(fileName)
if err == NotFoundErr {
@ -89,7 +79,14 @@ func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) {
return
}
encoded, err := createTorrent(fileName, filePath, r)
f, err := fileBackend.Open(fileName)
if err != nil {
oopsHandler(c, w, r, RespHTML, "Could not create torrent.")
return
}
defer f.Close()
encoded, err := createTorrent(fileName, f, r)
if err != nil {
oopsHandler(c, w, r, RespHTML, "Could not create torrent.")
return

17
torrent_test.go
View File

@ -2,6 +2,7 @@ package main
import (
"fmt"
"os"
"testing"
"github.com/zeebo/bencode"
@ -11,7 +12,13 @@ func TestCreateTorrent(t *testing.T) {
fileName := "server.go"
var decoded Torrent
encoded, err := createTorrent(fileName, fileName, nil)
f, err := os.Open("server.go")
if err != nil {
t.Fatal(err)
}
defer f.Close()
encoded, err := createTorrent(fileName, f, nil)
if err != nil {
t.Fatal(err)
}
@ -47,7 +54,13 @@ func TestCreateTorrent(t *testing.T) {
func TestCreateTorrentWithImage(t *testing.T) {
var decoded Torrent
encoded, err := createTorrent("test.jpg", "static/images/404.jpg", nil)
f, err := os.Open("static/images/404.jpg")
if err != nil {
t.Fatal(err)
}
defer f.Close()
encoded, err := createTorrent("test.jpg", f, nil)
if err != nil {
t.Fatal(err)
}

32
upload.go
View File

@ -8,7 +8,6 @@ import (
"io"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"regexp"
@ -232,9 +231,8 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
upload.Filename = strings.Join([]string{barename, extension}, ".")
_, err = os.Stat(path.Join(Config.filesDir, upload.Filename))
fileexists, _ := fileBackend.Exists(upload.Filename)
fileexists := err == nil
// Check if the delete key matches, in which case overwrite
if fileexists {
metad, merr := metadataRead(upload.Filename)
@ -254,20 +252,13 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
}
upload.Filename = strings.Join([]string{barename, extension}, ".")
_, err = os.Stat(path.Join(Config.filesDir, upload.Filename))
fileexists = err == nil
fileexists, err = fileBackend.Exists(upload.Filename)
}
if fileBlacklist[strings.ToLower(upload.Filename)] {
return upload, errors.New("Prohibited filename")
}
dst, err := os.Create(path.Join(Config.filesDir, upload.Filename))
if err != nil {
return
}
defer dst.Close()
// Get the rest of the metadata needed for storage
var expiry time.Time
if upReq.expiry == 0 {
@ -276,29 +267,22 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
expiry = time.Now().Add(upReq.expiry)
}
bytes, err := io.Copy(dst, io.MultiReader(bytes.NewReader(header), upReq.src))
if bytes == 0 {
os.Remove(path.Join(Config.filesDir, upload.Filename))
return upload, errors.New("Empty file")
} else if err != nil {
os.Remove(path.Join(Config.filesDir, upload.Filename))
return
bytes, err := fileBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src))
if err != nil {
return upload, err
} else if bytes > Config.maxSize {
os.Remove(path.Join(Config.filesDir, upload.Filename))
fileBackend.Delete(upload.Filename)
return upload, errors.New("File too large")
}
upload.Metadata, err = generateMetadata(upload.Filename, expiry, upReq.deletionKey)
if err != nil {
os.Remove(path.Join(Config.filesDir, upload.Filename))
os.Remove(path.Join(Config.metaDir, upload.Filename))
fileBackend.Delete(upload.Filename)
return
}
err = metadataWrite(upload.Filename, &upload.Metadata)
if err != nil {
os.Remove(path.Join(Config.filesDir, upload.Filename))
os.Remove(path.Join(Config.metaDir, upload.Filename))
fileBackend.Delete(upload.Filename)
return
}
return