applications/linx-server
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Metadata holds mimetype, sha256sum, archiveFiles

Browse Source
This commit is contained in:
andreimarcu 2015-10-07 22:45:34 -04:00
parent d05f0b645b
commit 6e33fe6ac8
7 changed files with 239 additions and 183 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
delete.go
View File

@ -23,14 +23,13 @@ func deleteHandler(c web.C, w http.ResponseWriter, r *http.Request) {
}
// Ensure delete key is correct
deleteKey, err := metadataGetDeleteKey(filename)
metadata, err := metadataRead(filename)
if err != nil {
unauthorizedHandler(c, w, r) // 401 - no metadata available
return
}
if deleteKey == requestKey {
if metadata.DeleteKey == requestKey {
fileDelErr := os.Remove(filePath)
metaDelErr := os.Remove(metaPath)

119
display.go
View File

@ -1,23 +1,16 @@
package main
import (
"archive/tar"
"archive/zip"
"compress/bzip2"
"compress/gzip"
"encoding/json"
"io"
"io/ioutil"
"net/http"
"os"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"bitbucket.org/taruti/mimemagic"
"github.com/dustin/go-humanize"
"github.com/flosch/pongo2"
"github.com/microcosm-cc/bluemonday"
@ -30,21 +23,24 @@ const maxDisplayFileSizeBytes = 1024 * 512
func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
fileName := c.URLParams["name"]
filePath := path.Join(Config.filesDir, fileName)
fileInfo, err := os.Stat(filePath)
if !fileExistsAndNotExpired(fileName) {
err := checkFile(fileName)
if err == NotFoundErr {
notFoundHandler(c, w, r)
return
}
expiry, _ := metadataGetExpiry(fileName)
var expiryHuman string
if expiry != neverExpire {
expiryHuman = humanize.RelTime(time.Now(), expiry, "", "")
metadata, err := metadataRead(fileName)
if err != nil {
oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.")
return
}
sizeHuman := humanize.Bytes(uint64(fileInfo.Size()))
var expiryHuman string
if metadata.Expiry != neverExpire {
expiryHuman = humanize.RelTime(time.Now(), metadata.Expiry, "", "")
}
sizeHuman := humanize.Bytes(uint64(metadata.Size))
extra := make(map[string]string)
files := []string{}
file, _ := os.Open(filePath)
defer file.Close()
@ -52,15 +48,15 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
header := make([]byte, 512)
file.Read(header)
mimetype := mimemagic.Match("", header)
extension := strings.TrimPrefix(filepath.Ext(fileName), ".")
if strings.EqualFold("application/json", r.Header.Get("Accept")) {
js, _ := json.Marshal(map[string]string{
"filename": fileName,
"mimetype": mimetype,
"expiry": strconv.FormatInt(expiry.Unix(), 10),
"size": strconv.FormatInt(fileInfo.Size(), 10),
"filename": fileName,
"expiry": strconv.FormatInt(metadata.Expiry.Unix(), 10),
"size": strconv.FormatInt(metadata.Size, 10),
"mimetype": metadata.Mimetype,
"sha256sum": metadata.Sha256sum,
})
w.Write(js)
return
@ -68,81 +64,20 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
var tpl *pongo2.Template
if strings.HasPrefix(mimetype, "image/") {
if strings.HasPrefix(metadata.Mimetype, "image/") {
tpl = Templates["display/image.html"]
} else if strings.HasPrefix(mimetype, "video/") {
} else if strings.HasPrefix(metadata.Mimetype, "video/") {
tpl = Templates["display/video.html"]
} else if strings.HasPrefix(mimetype, "audio/") {
} else if strings.HasPrefix(metadata.Mimetype, "audio/") {
tpl = Templates["display/audio.html"]
} else if mimetype == "application/pdf" {
} else if metadata.Mimetype == "application/pdf" {
tpl = Templates["display/pdf.html"]
} else if mimetype == "application/x-tar" {
f, _ := os.Open(filePath)
defer f.Close()
tReadr := tar.NewReader(f)
for {
header, err := tReadr.Next()
if err == io.EOF || err != nil {
break
}
if header.Typeflag == tar.TypeDir || header.Typeflag == tar.TypeReg {
files = append(files, header.Name)
}
}
sort.Strings(files)
} else if mimetype == "application/x-gzip" {
f, _ := os.Open(filePath)
defer f.Close()
gzf, err := gzip.NewReader(f)
if err == nil {
tReadr := tar.NewReader(gzf)
for {
header, err := tReadr.Next()
if err == io.EOF || err != nil {
break
}
if header.Typeflag == tar.TypeDir || header.Typeflag == tar.TypeReg {
files = append(files, header.Name)
}
}
sort.Strings(files)
}
} else if mimetype == "application/x-bzip" {
f, _ := os.Open(filePath)
defer f.Close()
bzf := bzip2.NewReader(f)
tReadr := tar.NewReader(bzf)
for {
header, err := tReadr.Next()
if err == io.EOF || err != nil {
break
}
if header.Typeflag == tar.TypeDir || header.Typeflag == tar.TypeReg {
files = append(files, header.Name)
}
}
sort.Strings(files)
} else if mimetype == "application/zip" {
f, _ := os.Open(filePath)
defer f.Close()
zf, err := zip.NewReader(f, fileInfo.Size())
if err == nil {
for _, f := range zf.File {
files = append(files, f.Name)
}
}
} else if supportedBinExtension(extension) {
if fileInfo.Size() < maxDisplayFileSizeBytes {
if metadata.Size < maxDisplayFileSizeBytes {
bytes, err := ioutil.ReadFile(filePath)
if err == nil {
extra["extension"] = extension
@ -152,7 +87,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
}
}
} else if extension == "md" {
if fileInfo.Size() < maxDisplayFileSizeBytes {
if metadata.Size < maxDisplayFileSizeBytes {
bytes, err := ioutil.ReadFile(filePath)
if err == nil {
unsafe := blackfriday.MarkdownCommon(bytes)
@ -170,12 +105,12 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
}
err = tpl.ExecuteWriter(pongo2.Context{
"mime": mimetype,
"mime": metadata.Mimetype,
"filename": fileName,
"size": sizeHuman,
"expiry": expiryHuman,
"extra": extra,
"files": files,
"files": metadata.ArchiveFiles,
}, w)
if err != nil {

10
expiry.go
View File

@ -13,7 +13,11 @@ func isTsExpired(ts time.Time) bool {
}
// Determine if the given filename is expired
func isFileExpired(filename string) bool {
exp, _ := metadataGetExpiry(filename)
return isTsExpired(exp)
func isFileExpired(filename string) (bool, error) {
metadata, err := metadataRead(filename)
if err != nil {
return false, err
}
return isTsExpired(metadata.Expiry), nil
}

21
fileserve.go
View File

@ -13,9 +13,13 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) {
fileName := c.URLParams["name"]
filePath := path.Join(Config.filesDir, fileName)
if !fileExistsAndNotExpired(fileName) {
err := checkFile(fileName)
if err == NotFoundErr {
notFoundHandler(c, w, r)
return
} else if err == BadMetadata {
oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.")
return
}
if !Config.allowHotlink {
@ -55,19 +59,24 @@ func staticHandler(c web.C, w http.ResponseWriter, r *http.Request) {
}
}
func fileExistsAndNotExpired(filename string) bool {
func checkFile(filename string) error {
filePath := path.Join(Config.filesDir, filename)
_, err := os.Stat(filePath)
if err != nil {
return false
return NotFoundErr
}
if isFileExpired(filename) {
expired, err := isFileExpired(filename)
if err != nil {
return err
}
if expired {
os.Remove(path.Join(Config.filesDir, filename))
os.Remove(path.Join(Config.metaDir, filename))
return false
return NotFoundErr
}
return true
return nil
}

218
meta.go
View File

@ -1,84 +1,188 @@
package main
import (
"bufio"
"archive/tar"
"archive/zip"
"compress/bzip2"
"compress/gzip"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"strconv"
"sort"
"time"
"bitbucket.org/taruti/mimemagic"
"github.com/dchest/uniuri"
)
// Write metadata from Upload struct to file
func metadataWrite(filename string, upload *Upload) error {
// Write metadata, overwriting if necessary
type MetadataJSON struct {
DeleteKey string `json:"delete_key"`
Sha256sum string `json:"sha256sum"`
Mimetype string `json:"mimetype"`
Size int64 `json:"size"`
Expiry int64 `json:"expiry"`
ArchiveFiles []string `json:"archive_files,omitempty"`
}
file, err := os.Create(path.Join(Config.metaDir, upload.Filename))
type Metadata struct {
DeleteKey string
Sha256sum string
Mimetype string
Size int64
Expiry time.Time
ArchiveFiles []string
}
var NotFoundErr = errors.New("File not found.")
var BadMetadata = errors.New("Corrupted metadata.")
func generateMetadata(fName string, exp time.Time, delKey string) (m Metadata, err error) {
file, err := os.Open(path.Join(Config.filesDir, fName))
fileInfo, err := os.Stat(path.Join(Config.filesDir, fName))
if err != nil {
return
}
defer file.Close()
m.Size = fileInfo.Size()
m.Expiry = exp
if delKey == "" {
m.DeleteKey = uniuri.NewLen(30)
} else {
m.DeleteKey = delKey
}
// Get first 512 bytes for mimetype detection
header := make([]byte, 512)
file.Read(header)
m.Mimetype = mimemagic.Match("", header)
// Compute the sha256sum
hasher := sha256.New()
file.Seek(0, 0)
_, err = io.Copy(hasher, file)
if err == nil {
m.Sha256sum = hex.EncodeToString(hasher.Sum(nil))
}
file.Seek(0, 0)
// If archive, grab list of filenames
if m.Mimetype == "application/x-tar" {
tReadr := tar.NewReader(file)
for {
hdr, err := tReadr.Next()
if err == io.EOF || err != nil {
break
}
if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg {
m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name)
}
}
sort.Strings(m.ArchiveFiles)
} else if m.Mimetype == "application/x-gzip" {
gzf, err := gzip.NewReader(file)
if err == nil {
tReadr := tar.NewReader(gzf)
for {
hdr, err := tReadr.Next()
if err == io.EOF || err != nil {
break
}
if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg {
m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name)
}
}
sort.Strings(m.ArchiveFiles)
}
} else if m.Mimetype == "application/x-bzip" {
bzf := bzip2.NewReader(file)
tReadr := tar.NewReader(bzf)
for {
hdr, err := tReadr.Next()
if err == io.EOF || err != nil {
break
}
if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg {
m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name)
}
}
sort.Strings(m.ArchiveFiles)
} else if m.Mimetype == "application/zip" {
zf, err := zip.NewReader(file, m.Size)
if err == nil {
for _, f := range zf.File {
m.ArchiveFiles = append(m.ArchiveFiles, f.Name)
}
}
sort.Strings(m.ArchiveFiles)
}
return
}
func metadataWrite(filename string, metadata *Metadata) error {
file, err := os.Create(path.Join(Config.metaDir, filename))
if err != nil {
return err
}
defer file.Close()
mjson := MetadataJSON{}
mjson.DeleteKey = metadata.DeleteKey
mjson.Mimetype = metadata.Mimetype
mjson.ArchiveFiles = metadata.ArchiveFiles
mjson.Sha256sum = metadata.Sha256sum
mjson.Expiry = metadata.Expiry.Unix()
mjson.Size = metadata.Size
byt, err := json.Marshal(mjson)
if err != nil {
return err
}
defer file.Close()
_, err = file.Write(byt)
if err != nil {
return err
}
w := bufio.NewWriter(file)
fmt.Fprintln(w, upload.Expiry.Unix())
fmt.Fprintln(w, upload.DeleteKey)
return w.Flush()
return nil
}
// Return list of strings from a filename's metadata source
func metadataRead(filename string) ([]string, error) {
file, err := os.Open(path.Join(Config.metaDir, filename))
func metadataRead(filename string) (metadata Metadata, err error) {
b, err := ioutil.ReadFile(path.Join(Config.metaDir, filename))
if err != nil {
return nil, err
// Metadata does not exist, generate one
newMData, err := generateMetadata(filename, neverExpire, "")
if err != nil {
return metadata, err
}
metadataWrite(filename, &newMData)
b, err = ioutil.ReadFile(path.Join(Config.metaDir, filename))
if err != nil {
return metadata, BadMetadata
}
}
defer file.Close()
var lines []string
scanner := bufio.NewScanner(file)
for scanner.Scan() {
lines = append(lines, scanner.Text())
}
return lines, scanner.Err()
}
func metadataGetExpiry(filename string) (expiry time.Time, err error) {
metadata, err := metadataRead(filename)
// XXX in this case it's up to the caller to determine proper behavior
// for a nonexistant metadata file or broken file
mjson := MetadataJSON{}
err = json.Unmarshal(b, &mjson)
if err != nil {
return
return metadata, BadMetadata
}
if len(metadata) < 1 {
err = errors.New("ERR: Metadata file does not contain expiry")
return
}
metadata.DeleteKey = mjson.DeleteKey
metadata.Mimetype = mjson.Mimetype
metadata.ArchiveFiles = mjson.ArchiveFiles
metadata.Sha256sum = mjson.Sha256sum
metadata.Expiry = time.Unix(mjson.Expiry, 0)
metadata.Size = mjson.Size
expirySecs, err := strconv.ParseInt(metadata[0], 10, 64)
expiry = time.Unix(expirySecs, 0)
return
}
func metadataGetDeleteKey(filename string) (string, error) {
metadata, err := metadataRead(filename)
if len(metadata) < 2 {
err := errors.New("ERR: Metadata file does not contain deletion key")
return "", err
}
if err != nil {
return "", err
} else {
return metadata[1], err
}
}

6
torrent.go
View File

@ -80,9 +80,13 @@ func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) {
fileName := c.URLParams["name"]
filePath := path.Join(Config.filesDir, fileName)
if !fileExistsAndNotExpired(fileName) {
err := checkFile(fileName)
if err == NotFoundErr {
notFoundHandler(c, w, r)
return
} else if err == BadMetadata {
oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.")
return
}
encoded, err := createTorrent(fileName, filePath)

43
upload.go
View File

@ -40,10 +40,8 @@ type UploadRequest struct {
// Metadata associated with a file as it would actually be stored
type Upload struct {
Filename string // Final filename on disk
Size int64
Expiry time.Time // Unix timestamp of expiry, 0=never
DeleteKey string // Deletion key, one generated if not provided
Filename string // Final filename on disk
Metadata Metadata
}
func uploadPostHandler(c web.C, w http.ResponseWriter, r *http.Request) {
@ -246,34 +244,35 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
defer dst.Close()
// Get the rest of the metadata needed for storage
var expiry time.Time
if upReq.expiry == 0 {
upload.Expiry = neverExpire
expiry = neverExpire
} else {
upload.Expiry = time.Now().Add(upReq.expiry)
expiry = time.Now().Add(upReq.expiry)
}
// If no delete key specified, pick a random one.
if upReq.deletionKey == "" {
upload.DeleteKey = uniuri.NewLen(30)
} else {
upload.DeleteKey = upReq.deletionKey
}
metadataWrite(upload.Filename, &upload)
bytes, err := io.Copy(dst, io.MultiReader(bytes.NewReader(header), upReq.src))
if bytes == 0 {
os.Remove(path.Join(Config.filesDir, upload.Filename))
os.Remove(path.Join(Config.metaDir, upload.Filename))
return upload, errors.New("Empty file")
} else if err != nil {
os.Remove(path.Join(Config.filesDir, upload.Filename))
os.Remove(path.Join(Config.metaDir, upload.Filename))
return
}
upload.Size = bytes
upload.Metadata, err = generateMetadata(upload.Filename, expiry, upReq.deletionKey)
if err != nil {
os.Remove(path.Join(Config.filesDir, upload.Filename))
os.Remove(path.Join(Config.metaDir, upload.Filename))
return
}
err = metadataWrite(upload.Filename, &upload.Metadata)
if err != nil {
os.Remove(path.Join(Config.filesDir, upload.Filename))
os.Remove(path.Join(Config.metaDir, upload.Filename))
return
}
return
}
@ -285,9 +284,11 @@ func generateJSONresponse(upload Upload) []byte {
js, _ := json.Marshal(map[string]string{
"url": Config.siteURL + upload.Filename,
"filename": upload.Filename,
"delete_key": upload.DeleteKey,
"expiry": strconv.FormatInt(upload.Expiry.Unix(), 10),
"size": strconv.FormatInt(upload.Size, 10),
"delete_key": upload.Metadata.DeleteKey,
"expiry": strconv.FormatInt(upload.Metadata.Expiry.Unix(), 10),
"size": strconv.FormatInt(upload.Metadata.Size, 10),
"mimetype": upload.Metadata.Mimetype,
"sha256sum": upload.Metadata.Sha256sum,
})
return js