Commit cc7a51ae authored by Dmitry's avatar Dmitry

aws s3 interaction

parent d3ed60b5
......@@ -6,6 +6,9 @@ version: "3.2"
services:
api:
image: golang:1.11
environment:
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
volumes:
- ./:/go/src/bmstu.codes/developers34/SBWeb
working_dir: /go/src/bmstu.codes/developers34/SBWeb
......@@ -18,6 +21,7 @@ services:
published: 8080
protocol: tcp
mode: host
redis:
image: redis:alpine
......@@ -25,12 +29,10 @@ services:
postgres:
volumes:
- ./pkg/db/data/init.sql:/docker-entrypoint-initdb.d/init.sql
# - /var/lib/postgresql/data:/var/lib/postgresql/data/pgdata
environment:
POSTGRES_DB: data
POSTGRES_INITDB_ARGS: "-l en_US.UTF-8 -E UTF8"
PGDATA: /var/lib/postgresql/data/pgdata
POSTGRES_USER: dataUSER
POSTGRES_PASSWORD: dataPASSWORD
#restart: always
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
image: postgres:alpine
\ No newline at end of file
package main
import (
"fmt"
"log"
"os"
"bmstu.codes/developers34/SBWeb/pkg/model"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
)
func initAmazon(m *model.Model) {
sess, err := session.NewSession(&aws.Config{
Region: aws.String("eu-central-1"),
})
if err != nil {
fmt.Println("First")
log.Fatal(err)
}
filename := "eee"
file, _ := os.Open("./AuthReq.PNG")
uploader := s3manager.NewUploader(sess)
output, err := uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String("search-build"),
Key: aws.String("/images/" + filename + ".png"),
Body: file,
ACL: aws.String("public-read"),
})
fmt.Println(output.Location)
}
func main() {
initAmazon(nil)
}
......@@ -53,4 +53,25 @@ curl -v --cookie "session_id=FpXOTUwLRYFkXJbHSXayUggXTQTCOa-cHk6lsCljwp0=; Expir
## Update user with avatar
curl -v -F "images=@/home/orangejohny/Downloads/PNG_transparency_demonstration_1.png" -F "first_name=ddddmitry" -F "last_name=karg" --cookie "session_id=rGrjp9pjNic6waxXZvbTQfW-5qUMDQws7hLlyv07-c8=; Expires=Mon, 05 Nov 2018 19:35:00 GMT" 127.0.0.1:8080/users/profile
\ No newline at end of file
curl -v -F "images=@/home/orangejohny/Downloads/PNG_transparency_demonstration_1.png" -F "first_name=ddddmitry" -F "last_name=karg" --cookie "session_id=rGrjp9pjNic6waxXZvbTQfW-5qUMDQws7hLlyv07-c8=; Expires=Mon, 05 Nov 2018 19:35:00 GMT" 127.0.0.1:8080/users/profile
AKIAICNNYQPNLYH7OBJQ
eAHWFKoqwBCFSzxdgQLBLVioA+uChaZMXbzbCeCv
export AWS_REGION=eu-west-2
sess, err := session.NewSession(&aws.Config{Region: aws.String("us-west-2")})
aws_access_key_id = AKIAICNNYQPNLYH7OBJQ
aws_secret_access_key = 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
export AWS_ACCESS_KEY_ID="AKIAICNNYQPNLYH7OBJQ"
export AWS_SECRET_ACCESS_KEY="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"
sess, err := session.NewSession(&aws.Config{
Region: aws.String("us-west-2"),
Credentials: credentials.NewStaticCredentials("AKID", "SECRET_KEY", "TOKEN"),
})
AKIAJMLQ6SRNXLZ3FA5Q
0sYGI/q5a4TFC9Kee4mHGNyFDvUAht53ikTYmogC
\ No newline at end of file
......@@ -313,7 +313,7 @@ func userCreatePage(m *model.Model) http.Handler {
// load images from request if it is possible
if isMultipartForm {
filenames, err := loadImages(r)
filenames, err := loadImages(r, m)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write(apiErrorHandle(checkImage, imgCreErr, err,
......@@ -584,10 +584,9 @@ func userUpdatePage(m *model.Model) http.Handler {
// check if image address not null and exists
if user.AvatarAddress.String != "" {
_, err = os.Stat("." + rDomain(user.AvatarAddress.String))
if os.IsNotExist(err) {
if !m.IsExist(user.AvatarAddress.String) {
w.WriteHeader(http.StatusBadRequest)
w.Write(apiErrorHandle(checkImage, imgExErr, err,
w.Write(apiErrorHandle(checkImage, imgExErr, errors.New("No such image"),
imgExMsg))
return
}
......@@ -600,13 +599,13 @@ func userUpdatePage(m *model.Model) http.Handler {
return
}
// delete existing avatar image
deleteImages([]string{userFromDB.AvatarAddress.String})
deleteImages([]string{userFromDB.AvatarAddress.String}, m)
}
// load images from request if image address is null and
// content-type is multipart/form-data
if isMultipartForm {
filenames, err := loadImages(r)
filenames, err := loadImages(r, m)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write(apiErrorHandle(checkImage, imgCreErr, err,
......@@ -673,7 +672,7 @@ func userDeletePage(m *model.Model) http.Handler {
w.Write(apiErrorHandle(connectProvider, getInfoDBErr, err, getInfoDBMsg))
return
}
deleteImages([]string{userFromDB.AvatarAddress.String})
deleteImages([]string{userFromDB.AvatarAddress.String}, m)
// TODO removing images of ads which were created by user
......@@ -767,7 +766,7 @@ func adCreatePage(m *model.Model) http.Handler {
ad.AdImages = nil
// load images from request if it is possible
if isMultipartForm {
filenames, err := loadImages(r)
filenames, err := loadImages(r, m)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write(apiErrorHandle(checkImage, imgCreErr, err,
......@@ -906,23 +905,22 @@ func adUpdatePage(m *model.Model) http.Handler {
// check if images are not null and exist
if ad.AdImages != nil {
for _, image := range ad.AdImages {
_, err = os.Stat("." + rDomain(image))
if os.IsNotExist(err) {
if !m.IsExist(image) {
w.WriteHeader(http.StatusBadRequest)
w.Write(apiErrorHandle(checkImage, imgExErr, err,
w.Write(apiErrorHandle(checkImage, imgExErr, errors.New("No such image"),
imgExMsg))
return
}
}
} else {
// TODO delete particular images of ad
deleteImages(adFromDatabase.AdImages)
deleteImages(adFromDatabase.AdImages, m)
}
// load images from request if existing images array is null and
// content-type is multipart/form-data
if isMultipartForm {
filenames, err := loadImages(r)
filenames, err := loadImages(r, m)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write(apiErrorHandle(checkImage, imgCreErr, err,
......@@ -988,7 +986,7 @@ func adDeletePage(m *model.Model) http.Handler {
}
// delete images of such ad
deleteImages(adFromDatabase.AdImages)
deleteImages(adFromDatabase.AdImages, m)
// remove ad from DB
_, err = m.RemoveAd(id)
......@@ -1003,19 +1001,27 @@ func adDeletePage(m *model.Model) http.Handler {
})
}
// images handling is implemented by aws s3 now
// sendImage handles */images/{filename} with method GET.
// It returns image with such filename if it's exists.
func sendImage(m *model.Model) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
filename, _ := mux.Vars(r)["filename"]
_, err := os.Stat("./images/" + filename)
if err != nil {
if !m.IsExist("/images/" + filename) {
w.WriteHeader(http.StatusBadRequest)
w.Write(apiErrorHandle(checkImage, noImgErr, err,
w.Write(apiErrorHandle(checkImage, noImgErr, errors.New("No such image"),
noImgMsg))
return
}
err := m.DownloadImage("/images/" + filename)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write(apiErrorHandle(connectProvider, "DownloadImgError", err,
"Can't access S3"))
return
}
http.ServeFile(w, r, "./images/"+filename)
os.Remove("./images/" + filename)
})
}
......@@ -10,9 +10,6 @@ import (
"bytes"
"errors"
"fmt"
"image"
"image/color"
"image/png"
"io"
"io/ioutil"
"log"
......@@ -193,6 +190,14 @@ type mockDataSM struct {
outputError error
}
type mockDataIM struct {
inputKey string
outputError error
outputBool bool
outputLocation string
}
// testCase represents one particular test case.
type testCase struct {
// flags for db
......@@ -219,6 +224,12 @@ type testCase struct {
isDeleteSession bool
isPrepareCheckConnSM bool
// flags for im
isExist bool
isDownload bool
isUpload bool
isDelete bool
// expect flags
isPrepareDB bool
isPrepareSM bool
......@@ -229,6 +240,7 @@ type testCase struct {
// data for functions of DB and SM
sm *mockDataSM
db *mockDataDB
im *mockDataIM
// expected response of server
expectedStatusCode int
......@@ -475,6 +487,7 @@ var testCases = []testCase{
{
isNewUser: true,
isPrepareDB: true,
isUpload: true,
request: func() *http.Request {
path := os.Getenv("CI_PROJECT_DIR") + "/docs/AuthReq.PNG"
file, err := os.Open(path)
......@@ -498,6 +511,9 @@ var testCases = []testCase{
r.Header.Set("Content-Type", writer.FormDataContentType())
return r
}(),
im: &mockDataIM{
outputError: nil,
},
db: &mockDataDB{
outputID: 17,
outputError: nil,
......@@ -1027,6 +1043,7 @@ var testCases = []testCase{
isPrepareDB: true,
isPrepareSM: true,
isGetUserWithIDImg: true,
isUpload: true,
request: func() *http.Request {
path := os.Getenv("CI_PROJECT_DIR") + "/docs/GeneralOverview.png"
file, err := os.Open(path)
......@@ -1049,6 +1066,9 @@ var testCases = []testCase{
r.Header.Set("Cookie", "session_id=123abc")
return r
}(),
im: &mockDataIM{
outputError: nil,
},
db: &mockDataDB{
inputIDimg: 17,
outputID: 17,
......@@ -1072,6 +1092,7 @@ var testCases = []testCase{
isSecondCheckSession: true,
isCheckSession: true,
isPrepareSM: true,
isExist: true,
request: func() *http.Request {
r, _ := http.NewRequest("POST", domain+"/users/profile",
strings.NewReader("first_name=Alex&last_name=Ivanov&avatar_address=blbabalbal"))
......@@ -1087,6 +1108,10 @@ var testCases = []testCase{
Login: "pet@animal.com",
},
},
im: &mockDataIM{
outputBool: false,
inputKey: "blbabalbal",
},
expectedStatusCode: 400,
},
{
......@@ -1442,6 +1467,7 @@ var testCases = []testCase{
isPrepareCheckConnSM: true,
isCheckSession: true,
isSecondCheckSession: true,
isUpload: true,
request: func() *http.Request {
path := os.Getenv("CI_PROJECT_DIR") + "/docs/AuthReq.PNG"
file, err := os.Open(path)
......@@ -1465,6 +1491,9 @@ var testCases = []testCase{
r.Header.Set("Cookie", "session_id=123abc")
return r
}(),
im: &mockDataIM{
outputError: nil,
},
db: &mockDataDB{
inputAd: adsInDB[15],
outputID: 15,
......@@ -1743,6 +1772,7 @@ var testCases = []testCase{
isPrepareCheckConnSM: true,
isCheckSession: true,
isSecondCheckSession: true,
isUpload: true,
request: func() *http.Request {
path := os.Getenv("CI_PROJECT_DIR") + "/docs/AuthReq.PNG"
file, err := os.Open(path)
......@@ -1766,6 +1796,9 @@ var testCases = []testCase{
r.Header.Set("Cookie", "session_id=123abc")
return r
}(),
im: &mockDataIM{
outputError: nil,
},
db: &mockDataDB{
inputID: 15,
inputAd: adsInDB[17],
......@@ -1791,12 +1824,17 @@ var testCases = []testCase{
isPrepareCheckConnSM: true,
isCheckSession: true,
isSecondCheckSession: true,
isExist: true,
request: func() *http.Request {
r, _ := http.NewRequest("POST", domain+"/ads/edit/15",
strings.NewReader("title=Building&city=Moscow&description_ad=Awesome&ad_images=/eee"))
r.Header.Set("Cookie", "session_id=123abc")
return r
}(),
im: &mockDataIM{
outputBool: false,
inputKey: "/eee",
},
db: &mockDataDB{
inputID: 15,
outputAd: adsInDB[16],
......@@ -2008,26 +2046,26 @@ var testCases = []testCase{
},
expectedStatusCode: 500,
},
{
request: func() *http.Request {
img := image.NewRGBA(image.Rect(0, 0, 100, 50))
img.Set(2, 3, color.RGBA{255, 0, 0, 255})
os.Mkdir("images", 0777)
f, _ := os.OpenFile("./images/image.png", os.O_WRONLY|os.O_CREATE, 0777)
png.Encode(f, img)
r, _ := http.NewRequest("GET", domain+"/images/image.png", nil)
f.Close()
return r
}(),
expectedStatusCode: 200,
},
{
request: func() *http.Request {
r, _ := http.NewRequest("GET", domain+"/images/image123.png", nil)
return r
}(),
expectedStatusCode: 400,
},
/* {
request: func() *http.Request {
img := image.NewRGBA(image.Rect(0, 0, 100, 50))
img.Set(2, 3, color.RGBA{255, 0, 0, 255})
os.Mkdir("images", 0777)
f, _ := os.OpenFile("./images/image.png", os.O_WRONLY|os.O_CREATE, 0777)
png.Encode(f, img)
r, _ := http.NewRequest("GET", domain+"/images/image.png", nil)
f.Close()
return r
}(),
expectedStatusCode: 200,
},
{
request: func() *http.Request {
r, _ := http.NewRequest("GET", domain+"/images/image123.png", nil)
return r
}(),
expectedStatusCode: 400,
}, */
}
func TestInterfaceOfAPI(t *testing.T) {
......@@ -2167,7 +2205,29 @@ func TestInterfaceOfAPI(t *testing.T) {
mockSM.EXPECT().IsConnected().Return(true)
}
tModel := model.New(mockDB, mockSM)
mockIM := mock_model.NewMockIM(ctrl)
if tCase.im != nil && tCase.isExist {
mockIM.EXPECT().IsExist(tCase.im.inputKey).
Return(tCase.im.outputBool).AnyTimes()
}
if tCase.im != nil && tCase.isUpload {
mockIM.EXPECT().UploadImage(gomock.Any(), gomock.Any()).
Return(tCase.im.outputLocation, tCase.im.outputError).AnyTimes()
}
if tCase.im != nil && tCase.isDelete {
mockIM.EXPECT().DeleteImage(gomock.Any()).
Return(tCase.im.outputError).AnyTimes()
}
if tCase.im != nil && tCase.isDownload {
mockIM.EXPECT().DownloadImage(gomock.Any()).
Return(tCase.im.outputError)
}
tModel := model.New(mockDB, mockSM, mockIM)
srv, ch := api.StartServer(api.Config{
Address: "localhost:49123",
......@@ -2269,11 +2329,12 @@ func TestMiddleware(t *testing.T) {
db := mock_model.NewMockDB(ctrl)
sm := mock_model.NewMockSM(ctrl)
im := mock_model.NewMockIM(ctrl)
sm.EXPECT().IsConnected().Return(false)
sm.EXPECT().TryReconnect().Return(nil)
m := model.New(db, sm)
m := model.New(db, sm, im)
srv, ch := api.StartServer(api.Config{
Address: "localhost:49123",
......@@ -2295,7 +2356,7 @@ func TestMiddleware(t *testing.T) {
sm.EXPECT().IsConnected().Return(false)
sm.EXPECT().TryReconnect().Return(errors.New("e"))
m = model.New(db, sm)
m = model.New(db, sm, im)
srv, ch = api.StartServer(api.Config{
Address: "localhost:49123",
......
// Code generated by MockGen. DO NOT EDIT.
// Source: bmstu.codes/developers34/SBWeb/pkg/model (interfaces: SM,DB)
// Source: bmstu.codes/developers34/SBWeb/pkg/model (interfaces: SM,DB,IM)
// Package mock_model is a generated GoMock package.
package mock_model
import (
reflect "reflect"
model "bmstu.codes/developers34/SBWeb/pkg/model"
gomock "github.com/golang/mock/gomock"
io "io"
reflect "reflect"
)
// MockSM is a mock of SM interface
......@@ -261,3 +261,75 @@ func (m *MockDB) RemoveUser(arg0 int64) (int64, error) {
func (mr *MockDBMockRecorder) RemoveUser(arg0 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveUser", reflect.TypeOf((*MockDB)(nil).RemoveUser), arg0)
}
// MockIM is a mock of IM interface
type MockIM struct {
ctrl *gomock.Controller
recorder *MockIMMockRecorder
}
// MockIMMockRecorder is the mock recorder for MockIM
type MockIMMockRecorder struct {
mock *MockIM
}
// NewMockIM creates a new mock instance
func NewMockIM(ctrl *gomock.Controller) *MockIM {
mock := &MockIM{ctrl: ctrl}
mock.recorder = &MockIMMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use
func (m *MockIM) EXPECT() *MockIMMockRecorder {
return m.recorder
}
// DeleteImage mocks base method
func (m *MockIM) DeleteImage(arg0 string) error {
ret := m.ctrl.Call(m, "DeleteImage", arg0)
ret0, _ := ret[0].(error)
return ret0
}
// DeleteImage indicates an expected call of DeleteImage
func (mr *MockIMMockRecorder) DeleteImage(arg0 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteImage", reflect.TypeOf((*MockIM)(nil).DeleteImage), arg0)
}
// DownloadImage mocks base method
func (m *MockIM) DownloadImage(arg0 string) error {
ret := m.ctrl.Call(m, "DownloadImage", arg0)
ret0, _ := ret[0].(error)
return ret0
}
// DownloadImage indicates an expected call of DownloadImage
func (mr *MockIMMockRecorder) DownloadImage(arg0 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DownloadImage", reflect.TypeOf((*MockIM)(nil).DownloadImage), arg0)
}
// IsExist mocks base method
func (m *MockIM) IsExist(arg0 string) bool {
ret := m.ctrl.Call(m, "IsExist", arg0)
ret0, _ := ret[0].(bool)
return ret0
}
// IsExist indicates an expected call of IsExist
func (mr *MockIMMockRecorder) IsExist(arg0 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsExist", reflect.TypeOf((*MockIM)(nil).IsExist), arg0)
}
// UploadImage mocks base method
func (m *MockIM) UploadImage(arg0 string, arg1 io.Reader) (string, error) {
ret := m.ctrl.Call(m, "UploadImage", arg0, arg1)
ret0, _ := ret[0].(string)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// UploadImage indicates an expected call of UploadImage
func (mr *MockIMMockRecorder) UploadImage(arg0, arg1 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UploadImage", reflect.TypeOf((*MockIM)(nil).UploadImage), arg0, arg1)
}
......@@ -42,7 +42,7 @@ func getIDfromCookie(m *model.Model, r *http.Request) int64 {
// loadImages process incoming request to upload images from it.
// ParseMultipartFrom must called before this function.
// It returns array of image's paths which were created.
func loadImages(r *http.Request) ([]string, error) {
func loadImages(r *http.Request, m *model.Model) ([]string, error) {
files := r.MultipartForm.File["images"]
filenames := make([]string, 0)
......@@ -94,6 +94,7 @@ func loadImages(r *http.Request) ([]string, error) {
io.Copy(hasher, bytes.NewReader(randBuf))
filename := hex.EncodeToString(hasher.Sum(nil))
dst, _ := os.Create("./images/" + filename + ".png")
defer os.Remove("./images/" + filename + ".png")
defer dst.Close()
// save image
......@@ -102,7 +103,13 @@ func loadImages(r *http.Request) ([]string, error) {
return nil, err
}
filenames = append(filenames, domain+"/images/"+filename+".png")
// use aws s3 for uploading
addr, err := m.UploadImage("/images/"+filename+".png", dst)
if err != nil {
return nil, err
}
filenames = append(filenames, addr)
// if we create or update user we need only one file
if strings.Contains(r.URL.Path, "/users/") {
......@@ -114,12 +121,12 @@ func loadImages(r *http.Request) ([]string, error) {
}
// deleteImages deletes files with filenames.
func deleteImages(filenames []string) error {
func deleteImages(filenames []string, m *model.Model) error {
for _, filename := range filenames {
if filename == "" {
break
}
err := os.Remove("." + filename)
err := m.DeleteImage(filename)
if err != nil {
return err
}
......@@ -127,10 +134,11 @@ func deleteImages(filenames []string) error {
return nil
}
// non need in this function since image handling by s3
// rDomain proccess string and deletes url
func rDomain(s string) string {
/* func rDomain(s string) string {
if strings.Contains(s, domain) {
return strings.Replace(s, domain, "", 1)
}
return s
}
} */
......@@ -13,5 +13,10 @@
"ReadTimeout": "10s",
"WriteTimeout": "10s",