functional_tests.go 379 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906790779087909791079117912791379147915791679177918791979207921792279237924792579267927792879297930793179327933793479357936793779387939794079417942794379447945794679477948794979507951795279537954795579567957795879597960796179627963796479657966796779687969797079717972797379747975797679777978797979807981798279837984798579867987798879897990799179927993799479957996799779987999800080018002800380048005800680078008800980108011801280138014801580168017801880198020802180228023802480258026802780288029803080318032803380348035803680378038803980408041804280438044804580468047804880498050805180528053805480558056805780588059806080618062806380648065806680678068806980708071807280738074807580768077807880798080808180828083808480858086808780888089809080918092809380948095809680978098809981008101810281038104810581068107810881098110811181128113811481158116811781188119812081218122812381248125812681278128812981308131813281338134813581368137813881398140814181428143814481458146814781488149815081518152815381548155815681578158815981608161816281638164816581668167816881698170817181728173817481758176817781788179818081818182818381848185818681878188818981908191819281938194819581968197819881998200820182028203820482058206820782088209821082118212821382148215821682178218821982208221822282238224822582268227822882298230823182328233823482358236823782388239824082418242824382448245824682478248824982508251825282538254825582568257825882598260826182628263826482658266826782688269827082718272827382748275827682778278827982808281828282838284828582868287828882898290829182928293829482958296829782988299830083018302830383048305830683078308830983108311831283138314831583168317831883198320832183228323832483258326832783288329833083318332833383348335833683378338833983408341834283438344834583468347834883498350835183528353835483558356835783588359836083618362836383648365836683678368836983708371837283738374837583768377837883798380838183828383838483858386838783888389839083918392839383948395839683978398839984008401840284038404840584068407840884098410841184128413841484158416841784188419842084218422842384248425842684278428842984308431843284338434843584368437843884398440844184428443844484458446844784488449845084518452845384548455845684578458845984608461846284638464846584668467846884698470847184728473847484758476847784788479848084818482848384848485848684878488848984908491849284938494849584968497849884998500850185028503850485058506850785088509851085118512851385148515851685178518851985208521852285238524852585268527852885298530853185328533853485358536853785388539854085418542854385448545854685478548854985508551855285538554855585568557855885598560856185628563856485658566856785688569857085718572857385748575857685778578857985808581858285838584858585868587858885898590859185928593859485958596859785988599860086018602860386048605860686078608860986108611861286138614861586168617861886198620862186228623862486258626862786288629863086318632863386348635863686378638863986408641864286438644864586468647864886498650865186528653865486558656865786588659866086618662866386648665866686678668866986708671867286738674867586768677867886798680868186828683868486858686868786888689869086918692869386948695869686978698869987008701870287038704870587068707870887098710871187128713871487158716871787188719872087218722872387248725872687278728872987308731873287338734873587368737873887398740874187428743874487458746874787488749875087518752875387548755875687578758875987608761876287638764876587668767876887698770877187728773877487758776877787788779878087818782878387848785878687878788878987908791879287938794879587968797879887998800880188028803880488058806880788088809881088118812881388148815881688178818881988208821882288238824882588268827882888298830883188328833883488358836883788388839884088418842884388448845884688478848884988508851885288538854885588568857885888598860886188628863886488658866886788688869887088718872887388748875887688778878887988808881888288838884888588868887888888898890889188928893889488958896889788988899890089018902890389048905890689078908890989108911891289138914891589168917891889198920892189228923892489258926892789288929893089318932893389348935893689378938893989408941894289438944894589468947894889498950895189528953895489558956895789588959896089618962896389648965896689678968896989708971897289738974897589768977897889798980898189828983898489858986898789888989899089918992899389948995899689978998899990009001900290039004900590069007900890099010901190129013901490159016901790189019902090219022902390249025902690279028902990309031903290339034903590369037903890399040904190429043904490459046904790489049905090519052905390549055905690579058905990609061906290639064906590669067906890699070907190729073907490759076907790789079908090819082908390849085908690879088908990909091909290939094909590969097909890999100910191029103910491059106910791089109911091119112911391149115911691179118911991209121912291239124912591269127912891299130913191329133913491359136913791389139914091419142914391449145914691479148914991509151915291539154915591569157915891599160916191629163916491659166916791689169917091719172917391749175917691779178917991809181918291839184918591869187918891899190919191929193919491959196919791989199920092019202920392049205920692079208920992109211921292139214921592169217921892199220922192229223922492259226922792289229923092319232923392349235923692379238923992409241924292439244924592469247924892499250925192529253925492559256925792589259926092619262926392649265926692679268926992709271927292739274927592769277927892799280928192829283928492859286928792889289929092919292929392949295929692979298929993009301930293039304930593069307930893099310931193129313931493159316931793189319932093219322932393249325932693279328932993309331933293339334933593369337933893399340934193429343934493459346934793489349935093519352935393549355935693579358935993609361936293639364936593669367936893699370937193729373937493759376937793789379938093819382938393849385938693879388938993909391939293939394939593969397939893999400940194029403940494059406940794089409941094119412941394149415941694179418941994209421942294239424942594269427942894299430943194329433943494359436943794389439944094419442944394449445944694479448944994509451945294539454945594569457945894599460946194629463946494659466946794689469947094719472947394749475947694779478947994809481948294839484948594869487948894899490949194929493949494959496949794989499950095019502950395049505950695079508950995109511951295139514951595169517951895199520952195229523952495259526952795289529953095319532953395349535953695379538953995409541954295439544954595469547954895499550955195529553955495559556955795589559956095619562956395649565956695679568956995709571957295739574957595769577957895799580958195829583958495859586958795889589959095919592959395949595959695979598959996009601960296039604960596069607960896099610961196129613961496159616961796189619962096219622962396249625962696279628962996309631963296339634963596369637963896399640964196429643964496459646964796489649965096519652965396549655965696579658965996609661966296639664966596669667966896699670967196729673967496759676967796789679968096819682968396849685968696879688968996909691969296939694969596969697969896999700970197029703970497059706970797089709971097119712971397149715971697179718971997209721972297239724972597269727972897299730973197329733973497359736973797389739974097419742974397449745974697479748974997509751975297539754975597569757975897599760976197629763976497659766976797689769977097719772977397749775977697779778977997809781978297839784978597869787978897899790979197929793979497959796979797989799980098019802980398049805980698079808980998109811981298139814981598169817981898199820982198229823982498259826982798289829983098319832983398349835983698379838983998409841984298439844984598469847984898499850985198529853985498559856985798589859986098619862986398649865986698679868986998709871987298739874987598769877987898799880988198829883988498859886988798889889989098919892989398949895989698979898989999009901990299039904990599069907990899099910991199129913991499159916991799189919992099219922992399249925992699279928992999309931993299339934993599369937993899399940994199429943994499459946994799489949995099519952995399549955995699579958995999609961996299639964996599669967996899699970997199729973997499759976997799789979998099819982998399849985998699879988998999909991999299939994999599969997999899991000010001100021000310004100051000610007100081000910010100111001210013100141001510016100171001810019100201002110022100231002410025100261002710028100291003010031100321003310034100351003610037100381003910040100411004210043100441004510046100471004810049100501005110052100531005410055100561005710058100591006010061100621006310064100651006610067100681006910070100711007210073100741007510076100771007810079100801008110082100831008410085100861008710088100891009010091100921009310094100951009610097100981009910100101011010210103101041010510106101071010810109101101011110112101131011410115101161011710118101191012010121101221012310124101251012610127101281012910130101311013210133101341013510136101371013810139101401014110142101431014410145101461014710148101491015010151101521015310154101551015610157101581015910160101611016210163101641016510166101671016810169101701017110172101731017410175101761017710178101791018010181101821018310184101851018610187101881018910190101911019210193101941019510196101971019810199102001020110202102031020410205102061020710208102091021010211102121021310214102151021610217102181021910220102211022210223102241022510226102271022810229102301023110232102331023410235102361023710238102391024010241102421024310244102451024610247102481024910250102511025210253102541025510256102571025810259102601026110262102631026410265102661026710268102691027010271102721027310274102751027610277102781027910280102811028210283102841028510286102871028810289102901029110292102931029410295102961029710298102991030010301103021030310304103051030610307103081030910310103111031210313103141031510316103171031810319103201032110322103231032410325103261032710328103291033010331103321033310334103351033610337103381033910340103411034210343103441034510346103471034810349103501035110352103531035410355103561035710358103591036010361103621036310364103651036610367103681036910370103711037210373103741037510376103771037810379103801038110382103831038410385103861038710388103891039010391103921039310394103951039610397103981039910400104011040210403104041040510406104071040810409104101041110412104131041410415104161041710418104191042010421104221042310424104251042610427104281042910430104311043210433104341043510436104371043810439104401044110442104431044410445104461044710448104491045010451104521045310454104551045610457104581045910460104611046210463104641046510466104671046810469104701047110472104731047410475104761047710478104791048010481104821048310484104851048610487104881048910490104911049210493104941049510496104971049810499105001050110502105031050410505105061050710508105091051010511105121051310514105151051610517105181051910520105211052210523105241052510526105271052810529105301053110532105331053410535105361053710538105391054010541105421054310544105451054610547105481054910550105511055210553105541055510556105571055810559105601056110562105631056410565105661056710568105691057010571105721057310574105751057610577105781057910580105811058210583105841058510586105871058810589105901059110592105931059410595105961059710598105991060010601106021060310604106051060610607106081060910610106111061210613106141061510616106171061810619106201062110622106231062410625106261062710628106291063010631106321063310634106351063610637106381063910640106411064210643106441064510646106471064810649106501065110652106531065410655106561065710658106591066010661106621066310664106651066610667106681066910670106711067210673106741067510676106771067810679106801068110682106831068410685106861068710688106891069010691106921069310694106951069610697106981069910700107011070210703107041070510706107071070810709107101071110712107131071410715107161071710718107191072010721107221072310724107251072610727107281072910730107311073210733107341073510736107371073810739107401074110742107431074410745107461074710748107491075010751107521075310754107551075610757107581075910760107611076210763107641076510766107671076810769107701077110772107731077410775107761077710778107791078010781107821078310784107851078610787107881078910790107911079210793107941079510796107971079810799108001080110802108031080410805108061080710808108091081010811108121081310814108151081610817108181081910820108211082210823108241082510826108271082810829108301083110832108331083410835108361083710838108391084010841108421084310844108451084610847108481084910850108511085210853108541085510856108571085810859108601086110862108631086410865108661086710868108691087010871108721087310874108751087610877108781087910880108811088210883108841088510886108871088810889108901089110892108931089410895108961089710898108991090010901109021090310904109051090610907109081090910910109111091210913109141091510916109171091810919109201092110922109231092410925109261092710928109291093010931109321093310934109351093610937109381093910940109411094210943109441094510946109471094810949109501095110952109531095410955109561095710958109591096010961109621096310964109651096610967109681096910970109711097210973109741097510976109771097810979109801098110982109831098410985109861098710988109891099010991109921099310994109951099610997109981099911000110011100211003110041100511006110071100811009110101101111012110131101411015110161101711018110191102011021110221102311024110251102611027110281102911030110311103211033110341103511036110371103811039110401104111042110431104411045110461104711048110491105011051110521105311054110551105611057110581105911060110611106211063110641106511066110671106811069110701107111072110731107411075110761107711078110791108011081110821108311084110851108611087110881108911090110911109211093110941109511096110971109811099111001110111102111031110411105111061110711108111091111011111111121111311114111151111611117111181111911120111211112211123111241112511126111271112811129111301113111132111331113411135111361113711138111391114011141111421114311144111451114611147111481114911150111511115211153111541115511156111571115811159111601116111162111631116411165111661116711168111691117011171111721117311174111751117611177111781117911180111811118211183111841118511186111871118811189111901119111192111931119411195111961119711198111991120011201112021120311204112051120611207112081120911210112111121211213112141121511216112171121811219112201122111222112231122411225112261122711228112291123011231112321123311234112351123611237112381123911240112411124211243112441124511246112471124811249112501125111252112531125411255112561125711258112591126011261112621126311264112651126611267112681126911270112711127211273112741127511276112771127811279112801128111282112831128411285112861128711288112891129011291112921129311294112951129611297112981129911300113011130211303113041130511306113071130811309113101131111312113131131411315113161131711318113191132011321113221132311324113251132611327113281132911330113311133211333113341133511336113371133811339113401134111342113431134411345113461134711348113491135011351113521135311354113551135611357113581135911360113611136211363113641136511366113671136811369113701137111372113731137411375113761137711378113791138011381113821138311384113851138611387113881138911390113911139211393113941139511396113971139811399114001140111402114031140411405114061140711408114091141011411114121141311414114151141611417114181141911420114211142211423114241142511426114271142811429114301143111432114331143411435114361143711438114391144011441114421144311444114451144611447114481144911450114511145211453114541145511456114571145811459114601146111462114631146411465114661146711468114691147011471114721147311474114751147611477114781147911480114811148211483114841148511486114871148811489114901149111492114931149411495114961149711498114991150011501115021150311504115051150611507115081150911510115111151211513115141151511516115171151811519115201152111522115231152411525115261152711528115291153011531115321153311534115351153611537115381153911540115411154211543115441154511546115471154811549115501155111552115531155411555115561155711558115591156011561115621156311564115651156611567115681156911570115711157211573115741157511576115771157811579115801158111582115831158411585115861158711588115891159011591115921159311594115951159611597115981159911600116011160211603116041160511606116071160811609116101161111612116131161411615116161161711618116191162011621116221162311624116251162611627116281162911630116311163211633116341163511636116371163811639116401164111642116431164411645116461164711648116491165011651116521165311654116551165611657116581165911660116611166211663116641166511666116671166811669116701167111672116731167411675116761167711678116791168011681116821168311684116851168611687116881168911690116911169211693116941169511696116971169811699117001170111702117031170411705117061170711708117091171011711117121171311714117151171611717117181171911720117211172211723117241172511726117271172811729117301173111732117331173411735117361173711738117391174011741117421174311744117451174611747117481174911750117511175211753117541175511756117571175811759117601176111762117631176411765117661176711768117691177011771117721177311774117751177611777117781177911780117811178211783117841178511786117871178811789117901179111792117931179411795117961179711798117991180011801118021180311804118051180611807118081180911810
  1. // +build mint
  2. /*
  3. * MinIO Go Library for Amazon S3 Compatible Cloud Storage
  4. * Copyright 2015-2020 MinIO, Inc.
  5. *
  6. * Licensed under the Apache License, Version 2.0 (the "License");
  7. * you may not use this file except in compliance with the License.
  8. * You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing, software
  13. * distributed under the License is distributed on an "AS IS" BASIS,
  14. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. * See the License for the specific language governing permissions and
  16. * limitations under the License.
  17. */
  18. package main
  19. import (
  20. "bytes"
  21. "context"
  22. "errors"
  23. "fmt"
  24. "hash/crc32"
  25. "io"
  26. "io/ioutil"
  27. "math/rand"
  28. "mime/multipart"
  29. "net/http"
  30. "net/url"
  31. "os"
  32. "path/filepath"
  33. "reflect"
  34. "runtime"
  35. "sort"
  36. "strconv"
  37. "strings"
  38. "sync"
  39. "time"
  40. "github.com/dustin/go-humanize"
  41. jsoniter "github.com/json-iterator/go"
  42. log "github.com/sirupsen/logrus"
  43. "github.com/minio/minio-go/v7"
  44. "github.com/minio/minio-go/v7/pkg/credentials"
  45. "github.com/minio/minio-go/v7/pkg/encrypt"
  46. "github.com/minio/minio-go/v7/pkg/notification"
  47. "github.com/minio/minio-go/v7/pkg/tags"
  48. )
  49. const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569"
  50. const (
  51. letterIdxBits = 6 // 6 bits to represent a letter index
  52. letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
  53. letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits
  54. )
  55. const (
  56. serverEndpoint = "SERVER_ENDPOINT"
  57. accessKey = "ACCESS_KEY"
  58. secretKey = "SECRET_KEY"
  59. enableHTTPS = "ENABLE_HTTPS"
  60. enableKMS = "ENABLE_KMS"
  61. )
  62. type mintJSONFormatter struct {
  63. }
  64. func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) {
  65. data := make(log.Fields, len(entry.Data))
  66. for k, v := range entry.Data {
  67. switch v := v.(type) {
  68. case error:
  69. // Otherwise errors are ignored by `encoding/json`
  70. // https://github.com/sirupsen/logrus/issues/137
  71. data[k] = v.Error()
  72. default:
  73. data[k] = v
  74. }
  75. }
  76. var json = jsoniter.ConfigCompatibleWithStandardLibrary
  77. serialized, err := json.Marshal(data)
  78. if err != nil {
  79. return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err)
  80. }
  81. return append(serialized, '\n'), nil
  82. }
  83. var readFull = func(r io.Reader, buf []byte) (n int, err error) {
  84. // ReadFull reads exactly len(buf) bytes from r into buf.
  85. // It returns the number of bytes copied and an error if
  86. // fewer bytes were read. The error is EOF only if no bytes
  87. // were read. If an EOF happens after reading some but not
  88. // all the bytes, ReadFull returns ErrUnexpectedEOF.
  89. // On return, n == len(buf) if and only if err == nil.
  90. // If r returns an error having read at least len(buf) bytes,
  91. // the error is dropped.
  92. for n < len(buf) && err == nil {
  93. var nn int
  94. nn, err = r.Read(buf[n:])
  95. // Some spurious io.Reader's return
  96. // io.ErrUnexpectedEOF when nn == 0
  97. // this behavior is undocumented
  98. // so we are on purpose not using io.ReadFull
  99. // implementation because this can lead
  100. // to custom handling, to avoid that
  101. // we simply modify the original io.ReadFull
  102. // implementation to avoid this issue.
  103. // io.ErrUnexpectedEOF with nn == 0 really
  104. // means that io.EOF
  105. if err == io.ErrUnexpectedEOF && nn == 0 {
  106. err = io.EOF
  107. }
  108. n += nn
  109. }
  110. if n >= len(buf) {
  111. err = nil
  112. } else if n > 0 && err == io.EOF {
  113. err = io.ErrUnexpectedEOF
  114. }
  115. return
  116. }
  117. func cleanEmptyEntries(fields log.Fields) log.Fields {
  118. cleanFields := log.Fields{}
  119. for k, v := range fields {
  120. if v != "" {
  121. cleanFields[k] = v
  122. }
  123. }
  124. return cleanFields
  125. }
  126. // log successful test runs
  127. func successLogger(testName string, function string, args map[string]interface{}, startTime time.Time) *log.Entry {
  128. // calculate the test case duration
  129. duration := time.Since(startTime)
  130. // log with the fields as per mint
  131. fields := log.Fields{"name": "minio-go: " + testName, "function": function, "args": args, "duration": duration.Nanoseconds() / 1000000, "status": "PASS"}
  132. return log.WithFields(cleanEmptyEntries(fields))
  133. }
  134. // As few of the features are not available in Gateway(s) currently, Check if err value is NotImplemented,
  135. // and log as NA in that case and continue execution. Otherwise log as failure and return
  136. func logError(testName string, function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) {
  137. // If server returns NotImplemented we assume it is gateway mode and hence log it as info and move on to next tests
  138. // Special case for ComposeObject API as it is implemented on client side and adds specific error details like `Error in upload-part-copy` in
  139. // addition to NotImplemented error returned from server
  140. if isErrNotImplemented(err) {
  141. ignoredLog(testName, function, args, startTime, message).Info()
  142. } else {
  143. failureLog(testName, function, args, startTime, alert, message, err).Fatal()
  144. }
  145. }
  146. // log failed test runs
  147. func failureLog(testName string, function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry {
  148. // calculate the test case duration
  149. duration := time.Since(startTime)
  150. var fields log.Fields
  151. // log with the fields as per mint
  152. if err != nil {
  153. fields = log.Fields{"name": "minio-go: " + testName, "function": function, "args": args,
  154. "duration": duration.Nanoseconds() / 1000000, "status": "FAIL", "alert": alert, "message": message, "error": err}
  155. } else {
  156. fields = log.Fields{"name": "minio-go: " + testName, "function": function, "args": args,
  157. "duration": duration.Nanoseconds() / 1000000, "status": "FAIL", "alert": alert, "message": message}
  158. }
  159. return log.WithFields(cleanEmptyEntries(fields))
  160. }
  161. // log not applicable test runs
  162. func ignoredLog(testName string, function string, args map[string]interface{}, startTime time.Time, alert string) *log.Entry {
  163. // calculate the test case duration
  164. duration := time.Since(startTime)
  165. // log with the fields as per mint
  166. fields := log.Fields{"name": "minio-go: " + testName, "function": function, "args": args,
  167. "duration": duration.Nanoseconds() / 1000000, "status": "NA", "alert": strings.Split(alert, " ")[0] + " is NotImplemented"}
  168. return log.WithFields(cleanEmptyEntries(fields))
  169. }
  170. // Delete objects in given bucket, recursively
  171. func cleanupBucket(bucketName string, c *minio.Client) error {
  172. // Create a done channel to control 'ListObjectsV2' go routine.
  173. doneCh := make(chan struct{})
  174. // Exit cleanly upon return.
  175. defer close(doneCh)
  176. // Iterate over all objects in the bucket via listObjectsV2 and delete
  177. for objCh := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{Recursive: true}) {
  178. if objCh.Err != nil {
  179. return objCh.Err
  180. }
  181. if objCh.Key != "" {
  182. err := c.RemoveObject(context.Background(), bucketName, objCh.Key, minio.RemoveObjectOptions{})
  183. if err != nil {
  184. return err
  185. }
  186. }
  187. }
  188. for objPartInfo := range c.ListIncompleteUploads(context.Background(), bucketName, "", true) {
  189. if objPartInfo.Err != nil {
  190. return objPartInfo.Err
  191. }
  192. if objPartInfo.Key != "" {
  193. err := c.RemoveIncompleteUpload(context.Background(), bucketName, objPartInfo.Key)
  194. if err != nil {
  195. return err
  196. }
  197. }
  198. }
  199. // objects are already deleted, clear the buckets now
  200. err := c.RemoveBucket(context.Background(), bucketName)
  201. if err != nil {
  202. return err
  203. }
  204. return err
  205. }
  206. func cleanupVersionedBucket(bucketName string, c *minio.Client) error {
  207. doneCh := make(chan struct{})
  208. defer close(doneCh)
  209. for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) {
  210. if obj.Err != nil {
  211. return obj.Err
  212. }
  213. if obj.Key != "" {
  214. err := c.RemoveObject(context.Background(), bucketName, obj.Key,
  215. minio.RemoveObjectOptions{VersionID: obj.VersionID, GovernanceBypass: true})
  216. if err != nil {
  217. return err
  218. }
  219. }
  220. }
  221. for objPartInfo := range c.ListIncompleteUploads(context.Background(), bucketName, "", true) {
  222. if objPartInfo.Err != nil {
  223. return objPartInfo.Err
  224. }
  225. if objPartInfo.Key != "" {
  226. err := c.RemoveIncompleteUpload(context.Background(), bucketName, objPartInfo.Key)
  227. if err != nil {
  228. return err
  229. }
  230. }
  231. }
  232. // objects are already deleted, clear the buckets now
  233. err := c.RemoveBucket(context.Background(), bucketName)
  234. if err != nil {
  235. for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) {
  236. log.Println("found", obj.Key, obj.VersionID)
  237. }
  238. return err
  239. }
  240. return err
  241. }
  242. func isErrNotImplemented(err error) bool {
  243. return minio.ToErrorResponse(err).Code == "NotImplemented"
  244. }
  245. func init() {
  246. // If server endpoint is not set, all tests default to
  247. // using https://play.min.io
  248. if os.Getenv(serverEndpoint) == "" {
  249. os.Setenv(serverEndpoint, "play.min.io")
  250. os.Setenv(accessKey, "Q3AM3UQ867SPQQA43P2F")
  251. os.Setenv(secretKey, "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG")
  252. os.Setenv(enableHTTPS, "1")
  253. }
  254. }
  255. var mintDataDir = os.Getenv("MINT_DATA_DIR")
  256. func getMintDataDirFilePath(filename string) (fp string) {
  257. if mintDataDir == "" {
  258. return
  259. }
  260. return filepath.Join(mintDataDir, filename)
  261. }
  262. func newRandomReader(seed, size int64) io.Reader {
  263. return io.LimitReader(rand.New(rand.NewSource(seed)), size)
  264. }
  265. func mustCrcReader(r io.Reader) uint32 {
  266. crc := crc32.NewIEEE()
  267. _, err := io.Copy(crc, r)
  268. if err != nil {
  269. panic(err)
  270. }
  271. return crc.Sum32()
  272. }
  273. func crcMatches(r io.Reader, want uint32) error {
  274. crc := crc32.NewIEEE()
  275. _, err := io.Copy(crc, r)
  276. if err != nil {
  277. panic(err)
  278. }
  279. got := crc.Sum32()
  280. if got != want {
  281. return fmt.Errorf("crc mismatch, want %x, got %x", want, got)
  282. }
  283. return nil
  284. }
  285. func crcMatchesName(r io.Reader, name string) error {
  286. want := dataFileCRC32[name]
  287. crc := crc32.NewIEEE()
  288. _, err := io.Copy(crc, r)
  289. if err != nil {
  290. panic(err)
  291. }
  292. got := crc.Sum32()
  293. if got != want {
  294. return fmt.Errorf("crc mismatch, want %x, got %x", want, got)
  295. }
  296. return nil
  297. }
  298. // read data from file if it exists or optionally create a buffer of particular size
  299. func getDataReader(fileName string) io.ReadCloser {
  300. if mintDataDir == "" {
  301. size := int64(dataFileMap[fileName])
  302. if _, ok := dataFileCRC32[fileName]; !ok {
  303. dataFileCRC32[fileName] = mustCrcReader(newRandomReader(size, size))
  304. }
  305. return ioutil.NopCloser(newRandomReader(size, size))
  306. }
  307. reader, _ := os.Open(getMintDataDirFilePath(fileName))
  308. if _, ok := dataFileCRC32[fileName]; !ok {
  309. dataFileCRC32[fileName] = mustCrcReader(reader)
  310. reader.Close()
  311. reader, _ = os.Open(getMintDataDirFilePath(fileName))
  312. }
  313. return reader
  314. }
  315. // randString generates random names and prepends them with a known prefix.
  316. func randString(n int, src rand.Source, prefix string) string {
  317. b := make([]byte, n)
  318. // A rand.Int63() generates 63 random bits, enough for letterIdxMax letters!
  319. for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
  320. if remain == 0 {
  321. cache, remain = src.Int63(), letterIdxMax
  322. }
  323. if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
  324. b[i] = letterBytes[idx]
  325. i--
  326. }
  327. cache >>= letterIdxBits
  328. remain--
  329. }
  330. return prefix + string(b[0:30-len(prefix)])
  331. }
  332. var dataFileMap = map[string]int{
  333. "datafile-0-b": 0,
  334. "datafile-1-b": 1,
  335. "datafile-1-kB": 1 * humanize.KiByte,
  336. "datafile-10-kB": 10 * humanize.KiByte,
  337. "datafile-33-kB": 33 * humanize.KiByte,
  338. "datafile-100-kB": 100 * humanize.KiByte,
  339. "datafile-1.03-MB": 1056 * humanize.KiByte,
  340. "datafile-1-MB": 1 * humanize.MiByte,
  341. "datafile-5-MB": 5 * humanize.MiByte,
  342. "datafile-6-MB": 6 * humanize.MiByte,
  343. "datafile-11-MB": 11 * humanize.MiByte,
  344. "datafile-65-MB": 65 * humanize.MiByte,
  345. "datafile-129-MB": 129 * humanize.MiByte,
  346. }
  347. var dataFileCRC32 = map[string]uint32{}
  348. func isFullMode() bool {
  349. return os.Getenv("MINT_MODE") == "full"
  350. }
  351. func getFuncName() string {
  352. return getFuncNameLoc(2)
  353. }
  354. func getFuncNameLoc(caller int) string {
  355. pc, _, _, _ := runtime.Caller(caller)
  356. return strings.TrimPrefix(runtime.FuncForPC(pc).Name(), "main.")
  357. }
  358. // Tests bucket re-create errors.
  359. func testMakeBucketError() {
  360. region := "eu-central-1"
  361. // initialize logging params
  362. startTime := time.Now()
  363. testName := getFuncName()
  364. function := "MakeBucket(bucketName, region)"
  365. // initialize logging params
  366. args := map[string]interface{}{
  367. "bucketName": "",
  368. "region": region,
  369. }
  370. // Seed random based on current time.
  371. rand.Seed(time.Now().Unix())
  372. // Instantiate new minio client object.
  373. c, err := minio.New(os.Getenv(serverEndpoint),
  374. &minio.Options{
  375. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  376. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  377. })
  378. if err != nil {
  379. logError(testName, function, args, startTime, "", "MinIO client creation failed", err)
  380. return
  381. }
  382. // Enable tracing, write to stderr.
  383. // c.TraceOn(os.Stderr)
  384. // Set user agent.
  385. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  386. // Generate a new random bucket name.
  387. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  388. args["bucketName"] = bucketName
  389. // Make a new bucket in 'eu-central-1'.
  390. if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err != nil {
  391. logError(testName, function, args, startTime, "", "MakeBucket Failed", err)
  392. return
  393. }
  394. defer cleanupBucket(bucketName, c)
  395. if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err == nil {
  396. logError(testName, function, args, startTime, "", "Bucket already exists", err)
  397. return
  398. }
  399. // Verify valid error response from server.
  400. if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" &&
  401. minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" {
  402. logError(testName, function, args, startTime, "", "Invalid error returned by server", err)
  403. return
  404. }
  405. successLogger(testName, function, args, startTime).Info()
  406. }
  407. func testMetadataSizeLimit() {
  408. startTime := time.Now()
  409. testName := getFuncName()
  410. function := "PutObject(bucketName, objectName, reader, objectSize, opts)"
  411. args := map[string]interface{}{
  412. "bucketName": "",
  413. "objectName": "",
  414. "opts.UserMetadata": "",
  415. }
  416. rand.Seed(startTime.Unix())
  417. // Instantiate new minio client object.
  418. c, err := minio.New(os.Getenv(serverEndpoint),
  419. &minio.Options{
  420. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  421. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  422. })
  423. if err != nil {
  424. logError(testName, function, args, startTime, "", "MinIO client creation failed", err)
  425. return
  426. }
  427. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  428. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  429. args["bucketName"] = bucketName
  430. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  431. args["objectName"] = objectName
  432. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  433. if err != nil {
  434. logError(testName, function, args, startTime, "", "Make bucket failed", err)
  435. return
  436. }
  437. defer cleanupBucket(bucketName, c)
  438. const HeaderSizeLimit = 8 * 1024
  439. const UserMetadataLimit = 2 * 1024
  440. // Meta-data greater than the 2 KB limit of AWS - PUT calls with this meta-data should fail
  441. metadata := make(map[string]string)
  442. metadata["X-Amz-Meta-Mint-Test"] = string(bytes.Repeat([]byte("m"), 1+UserMetadataLimit-len("X-Amz-Meta-Mint-Test")))
  443. args["metadata"] = fmt.Sprint(metadata)
  444. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata})
  445. if err == nil {
  446. logError(testName, function, args, startTime, "", "Created object with user-defined metadata exceeding metadata size limits", nil)
  447. return
  448. }
  449. // Meta-data (headers) greater than the 8 KB limit of AWS - PUT calls with this meta-data should fail
  450. metadata = make(map[string]string)
  451. metadata["X-Amz-Mint-Test"] = string(bytes.Repeat([]byte("m"), 1+HeaderSizeLimit-len("X-Amz-Mint-Test")))
  452. args["metadata"] = fmt.Sprint(metadata)
  453. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata})
  454. if err == nil {
  455. logError(testName, function, args, startTime, "", "Created object with headers exceeding header size limits", nil)
  456. return
  457. }
  458. successLogger(testName, function, args, startTime).Info()
  459. }
  460. // Tests various bucket supported formats.
  461. func testMakeBucketRegions() {
  462. region := "eu-central-1"
  463. // initialize logging params
  464. startTime := time.Now()
  465. testName := getFuncName()
  466. function := "MakeBucket(bucketName, region)"
  467. // initialize logging params
  468. args := map[string]interface{}{
  469. "bucketName": "",
  470. "region": region,
  471. }
  472. // Seed random based on current time.
  473. rand.Seed(time.Now().Unix())
  474. // Instantiate new minio client object.
  475. c, err := minio.New(os.Getenv(serverEndpoint),
  476. &minio.Options{
  477. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  478. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  479. })
  480. if err != nil {
  481. logError(testName, function, args, startTime, "", "MinIO client creation failed", err)
  482. return
  483. }
  484. // Enable tracing, write to stderr.
  485. // c.TraceOn(os.Stderr)
  486. // Set user agent.
  487. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  488. // Generate a new random bucket name.
  489. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  490. args["bucketName"] = bucketName
  491. // Make a new bucket in 'eu-central-1'.
  492. if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err != nil {
  493. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  494. return
  495. }
  496. // Delete all objects and buckets
  497. if err = cleanupBucket(bucketName, c); err != nil {
  498. logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
  499. return
  500. }
  501. // Make a new bucket with '.' in its name, in 'us-west-2'. This
  502. // request is internally staged into a path style instead of
  503. // virtual host style.
  504. region = "us-west-2"
  505. args["region"] = region
  506. if err = c.MakeBucket(context.Background(), bucketName+".withperiod", minio.MakeBucketOptions{Region: region}); err != nil {
  507. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  508. return
  509. }
  510. // Delete all objects and buckets
  511. if err = cleanupBucket(bucketName+".withperiod", c); err != nil {
  512. logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
  513. return
  514. }
  515. successLogger(testName, function, args, startTime).Info()
  516. }
  517. // Test PutObject using a large data to trigger multipart readat
  518. func testPutObjectReadAt() {
  519. // initialize logging params
  520. startTime := time.Now()
  521. testName := getFuncName()
  522. function := "PutObject(bucketName, objectName, reader, opts)"
  523. args := map[string]interface{}{
  524. "bucketName": "",
  525. "objectName": "",
  526. "opts": "objectContentType",
  527. }
  528. // Seed random based on current time.
  529. rand.Seed(time.Now().Unix())
  530. // Instantiate new minio client object.
  531. c, err := minio.New(os.Getenv(serverEndpoint),
  532. &minio.Options{
  533. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  534. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  535. })
  536. if err != nil {
  537. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  538. return
  539. }
  540. // Enable tracing, write to stderr.
  541. // c.TraceOn(os.Stderr)
  542. // Set user agent.
  543. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  544. // Generate a new random bucket name.
  545. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  546. args["bucketName"] = bucketName
  547. // Make a new bucket.
  548. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  549. if err != nil {
  550. logError(testName, function, args, startTime, "", "Make bucket failed", err)
  551. return
  552. }
  553. defer cleanupBucket(bucketName, c)
  554. bufSize := dataFileMap["datafile-129-MB"]
  555. var reader = getDataReader("datafile-129-MB")
  556. defer reader.Close()
  557. // Save the data
  558. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  559. args["objectName"] = objectName
  560. // Object content type
  561. objectContentType := "binary/octet-stream"
  562. args["objectContentType"] = objectContentType
  563. _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: objectContentType})
  564. if err != nil {
  565. logError(testName, function, args, startTime, "", "PutObject failed", err)
  566. return
  567. }
  568. // Read the data back
  569. r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  570. if err != nil {
  571. logError(testName, function, args, startTime, "", "Get Object failed", err)
  572. return
  573. }
  574. st, err := r.Stat()
  575. if err != nil {
  576. logError(testName, function, args, startTime, "", "Stat Object failed", err)
  577. return
  578. }
  579. if st.Size != int64(bufSize) {
  580. logError(testName, function, args, startTime, "", fmt.Sprintf("Number of bytes in stat does not match, expected %d got %d", bufSize, st.Size), err)
  581. return
  582. }
  583. if st.ContentType != objectContentType && st.ContentType != "application/octet-stream" {
  584. logError(testName, function, args, startTime, "", "Content types don't match", err)
  585. return
  586. }
  587. if err := crcMatchesName(r, "datafile-129-MB"); err != nil {
  588. logError(testName, function, args, startTime, "", "data CRC check failed", err)
  589. return
  590. }
  591. if err := r.Close(); err != nil {
  592. logError(testName, function, args, startTime, "", "Object Close failed", err)
  593. return
  594. }
  595. if err := r.Close(); err == nil {
  596. logError(testName, function, args, startTime, "", "Object is already closed, didn't return error on Close", err)
  597. return
  598. }
  599. successLogger(testName, function, args, startTime).Info()
  600. }
  601. func testListObjectVersions() {
  602. // initialize logging params
  603. startTime := time.Now()
  604. testName := getFuncName()
  605. function := "ListObjectVersions(bucketName, prefix, recursive)"
  606. args := map[string]interface{}{
  607. "bucketName": "",
  608. "prefix": "",
  609. "recursive": "",
  610. }
  611. // Seed random based on current time.
  612. rand.Seed(time.Now().Unix())
  613. // Instantiate new minio client object.
  614. c, err := minio.New(os.Getenv(serverEndpoint),
  615. &minio.Options{
  616. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  617. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  618. })
  619. if err != nil {
  620. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  621. return
  622. }
  623. // Enable tracing, write to stderr.
  624. // c.TraceOn(os.Stderr)
  625. // Set user agent.
  626. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  627. // Generate a new random bucket name.
  628. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  629. args["bucketName"] = bucketName
  630. // Make a new bucket.
  631. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
  632. if err != nil {
  633. logError(testName, function, args, startTime, "", "Make bucket failed", err)
  634. return
  635. }
  636. err = c.EnableVersioning(context.Background(), bucketName)
  637. if err != nil {
  638. logError(testName, function, args, startTime, "", "Enable versioning failed", err)
  639. return
  640. }
  641. // Save the data
  642. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  643. args["objectName"] = objectName
  644. bufSize := dataFileMap["datafile-10-kB"]
  645. var reader = getDataReader("datafile-10-kB")
  646. _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{})
  647. if err != nil {
  648. logError(testName, function, args, startTime, "", "PutObject failed", err)
  649. return
  650. }
  651. reader.Close()
  652. bufSize = dataFileMap["datafile-1-b"]
  653. reader = getDataReader("datafile-1-b")
  654. _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{})
  655. if err != nil {
  656. logError(testName, function, args, startTime, "", "PutObject failed", err)
  657. return
  658. }
  659. reader.Close()
  660. err = c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{})
  661. if err != nil {
  662. logError(testName, function, args, startTime, "", "Unexpected object deletion", err)
  663. return
  664. }
  665. var deleteMarkers, versions int
  666. objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
  667. for info := range objectsInfo {
  668. if info.Err != nil {
  669. logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
  670. return
  671. }
  672. if info.Key != objectName {
  673. logError(testName, function, args, startTime, "", "Unexpected object name in listing objects", nil)
  674. return
  675. }
  676. if info.VersionID == "" {
  677. logError(testName, function, args, startTime, "", "Unexpected version id in listing objects", nil)
  678. return
  679. }
  680. if info.IsDeleteMarker {
  681. deleteMarkers++
  682. if !info.IsLatest {
  683. logError(testName, function, args, startTime, "", "Unexpected IsLatest field in listing objects", nil)
  684. return
  685. }
  686. } else {
  687. versions++
  688. }
  689. }
  690. if deleteMarkers != 1 {
  691. logError(testName, function, args, startTime, "", "Unexpected number of DeleteMarker elements in listing objects", nil)
  692. return
  693. }
  694. if versions != 2 {
  695. logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil)
  696. return
  697. }
  698. // Delete all objects and their versions as long as the bucket itself
  699. if err = cleanupVersionedBucket(bucketName, c); err != nil {
  700. logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
  701. return
  702. }
  703. successLogger(testName, function, args, startTime).Info()
  704. }
  705. func testStatObjectWithVersioning() {
  706. // initialize logging params
  707. startTime := time.Now()
  708. testName := getFuncName()
  709. function := "StatObject"
  710. args := map[string]interface{}{}
  711. // Seed random based on current time.
  712. rand.Seed(time.Now().Unix())
  713. // Instantiate new minio client object.
  714. c, err := minio.New(os.Getenv(serverEndpoint),
  715. &minio.Options{
  716. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  717. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  718. })
  719. if err != nil {
  720. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  721. return
  722. }
  723. // Enable tracing, write to stderr.
  724. // c.TraceOn(os.Stderr)
  725. // Set user agent.
  726. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  727. // Generate a new random bucket name.
  728. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  729. args["bucketName"] = bucketName
  730. // Make a new bucket.
  731. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
  732. if err != nil {
  733. logError(testName, function, args, startTime, "", "Make bucket failed", err)
  734. return
  735. }
  736. err = c.EnableVersioning(context.Background(), bucketName)
  737. if err != nil {
  738. logError(testName, function, args, startTime, "", "Enable versioning failed", err)
  739. return
  740. }
  741. // Save the data
  742. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  743. args["objectName"] = objectName
  744. bufSize := dataFileMap["datafile-10-kB"]
  745. var reader = getDataReader("datafile-10-kB")
  746. _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{})
  747. if err != nil {
  748. logError(testName, function, args, startTime, "", "PutObject failed", err)
  749. return
  750. }
  751. reader.Close()
  752. bufSize = dataFileMap["datafile-1-b"]
  753. reader = getDataReader("datafile-1-b")
  754. _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{})
  755. if err != nil {
  756. logError(testName, function, args, startTime, "", "PutObject failed", err)
  757. return
  758. }
  759. reader.Close()
  760. objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
  761. var results []minio.ObjectInfo
  762. for info := range objectsInfo {
  763. if info.Err != nil {
  764. logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
  765. return
  766. }
  767. results = append(results, info)
  768. }
  769. if len(results) != 2 {
  770. logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil)
  771. return
  772. }
  773. for i := 0; i < len(results); i++ {
  774. opts := minio.StatObjectOptions{VersionID: results[i].VersionID}
  775. statInfo, err := c.StatObject(context.Background(), bucketName, objectName, opts)
  776. if err != nil {
  777. logError(testName, function, args, startTime, "", "error during HEAD object", err)
  778. return
  779. }
  780. if statInfo.VersionID == "" || statInfo.VersionID != results[i].VersionID {
  781. logError(testName, function, args, startTime, "", "error during HEAD object, unexpected version id", err)
  782. return
  783. }
  784. if statInfo.ETag != results[i].ETag {
  785. logError(testName, function, args, startTime, "", "error during HEAD object, unexpected ETag", err)
  786. return
  787. }
  788. if statInfo.LastModified.Unix() != results[i].LastModified.Unix() {
  789. logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Last-Modified", err)
  790. return
  791. }
  792. if statInfo.Size != results[i].Size {
  793. logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Content-Length", err)
  794. return
  795. }
  796. }
  797. // Delete all objects and their versions as long as the bucket itself
  798. if err = cleanupVersionedBucket(bucketName, c); err != nil {
  799. logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
  800. return
  801. }
  802. successLogger(testName, function, args, startTime).Info()
  803. }
  804. func testGetObjectWithVersioning() {
  805. // initialize logging params
  806. startTime := time.Now()
  807. testName := getFuncName()
  808. function := "GetObject()"
  809. args := map[string]interface{}{}
  810. // Seed random based on current time.
  811. rand.Seed(time.Now().Unix())
  812. // Instantiate new minio client object.
  813. c, err := minio.New(os.Getenv(serverEndpoint),
  814. &minio.Options{
  815. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  816. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  817. })
  818. if err != nil {
  819. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  820. return
  821. }
  822. // Enable tracing, write to stderr.
  823. // c.TraceOn(os.Stderr)
  824. // Set user agent.
  825. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  826. // Generate a new random bucket name.
  827. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  828. args["bucketName"] = bucketName
  829. // Make a new bucket.
  830. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
  831. if err != nil {
  832. logError(testName, function, args, startTime, "", "Make bucket failed", err)
  833. return
  834. }
  835. err = c.EnableVersioning(context.Background(), bucketName)
  836. if err != nil {
  837. logError(testName, function, args, startTime, "", "Enable versioning failed", err)
  838. return
  839. }
  840. // Save the data
  841. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  842. args["objectName"] = objectName
  843. // Save the contents of datafiles to check with GetObject() reader output later
  844. var buffers [][]byte
  845. var testFiles = []string{"datafile-1-b", "datafile-10-kB"}
  846. for _, testFile := range testFiles {
  847. r := getDataReader(testFile)
  848. buf, err := ioutil.ReadAll(r)
  849. if err != nil {
  850. logError(testName, function, args, startTime, "", "unexpected failure", err)
  851. return
  852. }
  853. r.Close()
  854. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{})
  855. if err != nil {
  856. logError(testName, function, args, startTime, "", "PutObject failed", err)
  857. return
  858. }
  859. buffers = append(buffers, buf)
  860. }
  861. objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
  862. var results []minio.ObjectInfo
  863. for info := range objectsInfo {
  864. if info.Err != nil {
  865. logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
  866. return
  867. }
  868. results = append(results, info)
  869. }
  870. if len(results) != 2 {
  871. logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil)
  872. return
  873. }
  874. sort.SliceStable(results, func(i, j int) bool {
  875. return results[i].Size < results[j].Size
  876. })
  877. sort.SliceStable(buffers, func(i, j int) bool {
  878. return len(buffers[i]) < len(buffers[j])
  879. })
  880. for i := 0; i < len(results); i++ {
  881. opts := minio.GetObjectOptions{VersionID: results[i].VersionID}
  882. reader, err := c.GetObject(context.Background(), bucketName, objectName, opts)
  883. if err != nil {
  884. logError(testName, function, args, startTime, "", "error during GET object", err)
  885. return
  886. }
  887. statInfo, err := reader.Stat()
  888. if err != nil {
  889. logError(testName, function, args, startTime, "", "error during calling reader.Stat()", err)
  890. return
  891. }
  892. if statInfo.ETag != results[i].ETag {
  893. logError(testName, function, args, startTime, "", "error during HEAD object, unexpected ETag", err)
  894. return
  895. }
  896. if statInfo.LastModified.Unix() != results[i].LastModified.Unix() {
  897. logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Last-Modified", err)
  898. return
  899. }
  900. if statInfo.Size != results[i].Size {
  901. logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Content-Length", err)
  902. return
  903. }
  904. tmpBuffer := bytes.NewBuffer([]byte{})
  905. _, err = io.Copy(tmpBuffer, reader)
  906. if err != nil {
  907. logError(testName, function, args, startTime, "", "unexpected io.Copy()", err)
  908. return
  909. }
  910. if !bytes.Equal(tmpBuffer.Bytes(), buffers[i]) {
  911. logError(testName, function, args, startTime, "", "unexpected content of GetObject()", err)
  912. return
  913. }
  914. }
  915. // Delete all objects and their versions as long as the bucket itself
  916. if err = cleanupVersionedBucket(bucketName, c); err != nil {
  917. logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
  918. return
  919. }
  920. successLogger(testName, function, args, startTime).Info()
  921. }
  922. func testPutObjectWithVersioning() {
  923. // initialize logging params
  924. startTime := time.Now()
  925. testName := getFuncName()
  926. function := "GetObject()"
  927. args := map[string]interface{}{}
  928. // Seed random based on current time.
  929. rand.Seed(time.Now().Unix())
  930. // Instantiate new minio client object.
  931. c, err := minio.New(os.Getenv(serverEndpoint),
  932. &minio.Options{
  933. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  934. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  935. })
  936. if err != nil {
  937. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  938. return
  939. }
  940. // Enable tracing, write to stderr.
  941. // c.TraceOn(os.Stderr)
  942. // Set user agent.
  943. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  944. // Generate a new random bucket name.
  945. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  946. args["bucketName"] = bucketName
  947. // Make a new bucket.
  948. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
  949. if err != nil {
  950. logError(testName, function, args, startTime, "", "Make bucket failed", err)
  951. return
  952. }
  953. err = c.EnableVersioning(context.Background(), bucketName)
  954. if err != nil {
  955. logError(testName, function, args, startTime, "", "Enable versioning failed", err)
  956. return
  957. }
  958. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  959. args["objectName"] = objectName
  960. const n = 10
  961. // Read input...
  962. // Save the data concurrently.
  963. var wg sync.WaitGroup
  964. wg.Add(n)
  965. var buffers = make([][]byte, n)
  966. var errs [n]error
  967. for i := 0; i < n; i++ {
  968. r := newRandomReader(int64((1<<20)*i+i), int64(i))
  969. buf, err := ioutil.ReadAll(r)
  970. if err != nil {
  971. logError(testName, function, args, startTime, "", "unexpected failure", err)
  972. return
  973. }
  974. buffers[i] = buf
  975. go func(i int) {
  976. defer wg.Done()
  977. _, errs[i] = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{PartSize: 5 << 20})
  978. }(i)
  979. }
  980. wg.Wait()
  981. for _, err := range errs {
  982. if err != nil {
  983. logError(testName, function, args, startTime, "", "PutObject failed", err)
  984. return
  985. }
  986. }
  987. objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
  988. var results []minio.ObjectInfo
  989. for info := range objectsInfo {
  990. if info.Err != nil {
  991. logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
  992. return
  993. }
  994. results = append(results, info)
  995. }
  996. if len(results) != n {
  997. logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil)
  998. return
  999. }
  1000. sort.Slice(results, func(i, j int) bool {
  1001. return results[i].Size < results[j].Size
  1002. })
  1003. sort.Slice(buffers, func(i, j int) bool {
  1004. return len(buffers[i]) < len(buffers[j])
  1005. })
  1006. for i := 0; i < len(results); i++ {
  1007. opts := minio.GetObjectOptions{VersionID: results[i].VersionID}
  1008. reader, err := c.GetObject(context.Background(), bucketName, objectName, opts)
  1009. if err != nil {
  1010. logError(testName, function, args, startTime, "", "error during GET object", err)
  1011. return
  1012. }
  1013. statInfo, err := reader.Stat()
  1014. if err != nil {
  1015. logError(testName, function, args, startTime, "", "error during calling reader.Stat()", err)
  1016. return
  1017. }
  1018. if statInfo.ETag != results[i].ETag {
  1019. logError(testName, function, args, startTime, "", "error during HEAD object, unexpected ETag", err)
  1020. return
  1021. }
  1022. if statInfo.LastModified.Unix() != results[i].LastModified.Unix() {
  1023. logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Last-Modified", err)
  1024. return
  1025. }
  1026. if statInfo.Size != results[i].Size {
  1027. logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Content-Length", err)
  1028. return
  1029. }
  1030. tmpBuffer := bytes.NewBuffer([]byte{})
  1031. _, err = io.Copy(tmpBuffer, reader)
  1032. if err != nil {
  1033. logError(testName, function, args, startTime, "", "unexpected io.Copy()", err)
  1034. return
  1035. }
  1036. if !bytes.Equal(tmpBuffer.Bytes(), buffers[i]) {
  1037. logError(testName, function, args, startTime, "", "unexpected content of GetObject()", err)
  1038. return
  1039. }
  1040. }
  1041. // Delete all objects and their versions as long as the bucket itself
  1042. if err = cleanupVersionedBucket(bucketName, c); err != nil {
  1043. logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
  1044. return
  1045. }
  1046. successLogger(testName, function, args, startTime).Info()
  1047. }
  1048. func testCopyObjectWithVersioning() {
  1049. // initialize logging params
  1050. startTime := time.Now()
  1051. testName := getFuncName()
  1052. function := "CopyObject()"
  1053. args := map[string]interface{}{}
  1054. // Seed random based on current time.
  1055. rand.Seed(time.Now().Unix())
  1056. // Instantiate new minio client object.
  1057. c, err := minio.New(os.Getenv(serverEndpoint),
  1058. &minio.Options{
  1059. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  1060. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  1061. })
  1062. if err != nil {
  1063. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  1064. return
  1065. }
  1066. // Enable tracing, write to stderr.
  1067. // c.TraceOn(os.Stderr)
  1068. // Set user agent.
  1069. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  1070. // Generate a new random bucket name.
  1071. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  1072. args["bucketName"] = bucketName
  1073. // Make a new bucket.
  1074. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
  1075. if err != nil {
  1076. logError(testName, function, args, startTime, "", "Make bucket failed", err)
  1077. return
  1078. }
  1079. err = c.EnableVersioning(context.Background(), bucketName)
  1080. if err != nil {
  1081. logError(testName, function, args, startTime, "", "Enable versioning failed", err)
  1082. return
  1083. }
  1084. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  1085. args["objectName"] = objectName
  1086. var testFiles = []string{"datafile-1-b", "datafile-10-kB"}
  1087. for _, testFile := range testFiles {
  1088. r := getDataReader(testFile)
  1089. buf, err := ioutil.ReadAll(r)
  1090. if err != nil {
  1091. logError(testName, function, args, startTime, "", "unexpected failure", err)
  1092. return
  1093. }
  1094. r.Close()
  1095. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{})
  1096. if err != nil {
  1097. logError(testName, function, args, startTime, "", "PutObject failed", err)
  1098. return
  1099. }
  1100. }
  1101. objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
  1102. var infos []minio.ObjectInfo
  1103. for info := range objectsInfo {
  1104. if info.Err != nil {
  1105. logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
  1106. return
  1107. }
  1108. infos = append(infos, info)
  1109. }
  1110. sort.Slice(infos, func(i, j int) bool {
  1111. return infos[i].Size < infos[j].Size
  1112. })
  1113. reader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{VersionID: infos[0].VersionID})
  1114. if err != nil {
  1115. logError(testName, function, args, startTime, "", "GetObject of the oldest version content failed", err)
  1116. return
  1117. }
  1118. oldestContent, err := ioutil.ReadAll(reader)
  1119. if err != nil {
  1120. logError(testName, function, args, startTime, "", "Reading the oldest object version failed", err)
  1121. return
  1122. }
  1123. // Copy Source
  1124. srcOpts := minio.CopySrcOptions{
  1125. Bucket: bucketName,
  1126. Object: objectName,
  1127. VersionID: infos[0].VersionID,
  1128. }
  1129. args["src"] = srcOpts
  1130. dstOpts := minio.CopyDestOptions{
  1131. Bucket: bucketName,
  1132. Object: objectName + "-copy",
  1133. }
  1134. args["dst"] = dstOpts
  1135. // Perform the Copy
  1136. if _, err = c.CopyObject(context.Background(), dstOpts, srcOpts); err != nil {
  1137. logError(testName, function, args, startTime, "", "CopyObject failed", err)
  1138. return
  1139. }
  1140. // Destination object
  1141. readerCopy, err := c.GetObject(context.Background(), bucketName, objectName+"-copy", minio.GetObjectOptions{})
  1142. if err != nil {
  1143. logError(testName, function, args, startTime, "", "GetObject failed", err)
  1144. return
  1145. }
  1146. defer readerCopy.Close()
  1147. newestContent, err := ioutil.ReadAll(readerCopy)
  1148. if err != nil {
  1149. logError(testName, function, args, startTime, "", "Reading from GetObject reader failed", err)
  1150. return
  1151. }
  1152. if len(newestContent) == 0 || !bytes.Equal(oldestContent, newestContent) {
  1153. logError(testName, function, args, startTime, "", "Unexpected destination object content", err)
  1154. return
  1155. }
  1156. // Delete all objects and their versions as long as the bucket itself
  1157. if err = cleanupVersionedBucket(bucketName, c); err != nil {
  1158. logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
  1159. return
  1160. }
  1161. successLogger(testName, function, args, startTime).Info()
  1162. }
  1163. func testConcurrentCopyObjectWithVersioning() {
  1164. // initialize logging params
  1165. startTime := time.Now()
  1166. testName := getFuncName()
  1167. function := "CopyObject()"
  1168. args := map[string]interface{}{}
  1169. // Seed random based on current time.
  1170. rand.Seed(time.Now().Unix())
  1171. // Instantiate new minio client object.
  1172. c, err := minio.New(os.Getenv(serverEndpoint),
  1173. &minio.Options{
  1174. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  1175. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  1176. })
  1177. if err != nil {
  1178. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  1179. return
  1180. }
  1181. // Enable tracing, write to stderr.
  1182. // c.TraceOn(os.Stderr)
  1183. // Set user agent.
  1184. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  1185. // Generate a new random bucket name.
  1186. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  1187. args["bucketName"] = bucketName
  1188. // Make a new bucket.
  1189. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
  1190. if err != nil {
  1191. logError(testName, function, args, startTime, "", "Make bucket failed", err)
  1192. return
  1193. }
  1194. err = c.EnableVersioning(context.Background(), bucketName)
  1195. if err != nil {
  1196. logError(testName, function, args, startTime, "", "Enable versioning failed", err)
  1197. return
  1198. }
  1199. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  1200. args["objectName"] = objectName
  1201. var testFiles = []string{"datafile-10-kB"}
  1202. for _, testFile := range testFiles {
  1203. r := getDataReader(testFile)
  1204. buf, err := ioutil.ReadAll(r)
  1205. if err != nil {
  1206. logError(testName, function, args, startTime, "", "unexpected failure", err)
  1207. return
  1208. }
  1209. r.Close()
  1210. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{})
  1211. if err != nil {
  1212. logError(testName, function, args, startTime, "", "PutObject failed", err)
  1213. return
  1214. }
  1215. }
  1216. objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
  1217. var infos []minio.ObjectInfo
  1218. for info := range objectsInfo {
  1219. if info.Err != nil {
  1220. logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
  1221. return
  1222. }
  1223. infos = append(infos, info)
  1224. }
  1225. sort.Slice(infos, func(i, j int) bool {
  1226. return infos[i].Size < infos[j].Size
  1227. })
  1228. reader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{VersionID: infos[0].VersionID})
  1229. if err != nil {
  1230. logError(testName, function, args, startTime, "", "GetObject of the oldest version content failed", err)
  1231. return
  1232. }
  1233. oldestContent, err := ioutil.ReadAll(reader)
  1234. if err != nil {
  1235. logError(testName, function, args, startTime, "", "Reading the oldest object version failed", err)
  1236. return
  1237. }
  1238. // Copy Source
  1239. srcOpts := minio.CopySrcOptions{
  1240. Bucket: bucketName,
  1241. Object: objectName,
  1242. VersionID: infos[0].VersionID,
  1243. }
  1244. args["src"] = srcOpts
  1245. dstOpts := minio.CopyDestOptions{
  1246. Bucket: bucketName,
  1247. Object: objectName + "-copy",
  1248. }
  1249. args["dst"] = dstOpts
  1250. // Perform the Copy concurrently
  1251. const n = 10
  1252. var wg sync.WaitGroup
  1253. wg.Add(n)
  1254. var errs [n]error
  1255. for i := 0; i < n; i++ {
  1256. go func(i int) {
  1257. defer wg.Done()
  1258. _, errs[i] = c.CopyObject(context.Background(), dstOpts, srcOpts)
  1259. }(i)
  1260. }
  1261. wg.Wait()
  1262. for _, err := range errs {
  1263. if err != nil {
  1264. logError(testName, function, args, startTime, "", "CopyObject failed", err)
  1265. return
  1266. }
  1267. }
  1268. objectsInfo = c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: false, Prefix: dstOpts.Object})
  1269. infos = []minio.ObjectInfo{}
  1270. for info := range objectsInfo {
  1271. // Destination object
  1272. readerCopy, err := c.GetObject(context.Background(), bucketName, objectName+"-copy", minio.GetObjectOptions{VersionID: info.VersionID})
  1273. if err != nil {
  1274. logError(testName, function, args, startTime, "", "GetObject failed", err)
  1275. return
  1276. }
  1277. defer readerCopy.Close()
  1278. newestContent, err := ioutil.ReadAll(readerCopy)
  1279. if err != nil {
  1280. logError(testName, function, args, startTime, "", "Reading from GetObject reader failed", err)
  1281. return
  1282. }
  1283. if len(newestContent) == 0 || !bytes.Equal(oldestContent, newestContent) {
  1284. logError(testName, function, args, startTime, "", "Unexpected destination object content", err)
  1285. return
  1286. }
  1287. infos = append(infos, info)
  1288. }
  1289. if len(infos) != n {
  1290. logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil)
  1291. return
  1292. }
  1293. // Delete all objects and their versions as long as the bucket itself
  1294. if err = cleanupVersionedBucket(bucketName, c); err != nil {
  1295. logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
  1296. return
  1297. }
  1298. successLogger(testName, function, args, startTime).Info()
  1299. }
  1300. func testComposeObjectWithVersioning() {
  1301. // initialize logging params
  1302. startTime := time.Now()
  1303. testName := getFuncName()
  1304. function := "ComposeObject()"
  1305. args := map[string]interface{}{}
  1306. // Seed random based on current time.
  1307. rand.Seed(time.Now().Unix())
  1308. // Instantiate new minio client object.
  1309. c, err := minio.New(os.Getenv(serverEndpoint),
  1310. &minio.Options{
  1311. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  1312. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  1313. })
  1314. if err != nil {
  1315. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  1316. return
  1317. }
  1318. // Enable tracing, write to stderr.
  1319. // c.TraceOn(os.Stderr)
  1320. // Set user agent.
  1321. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  1322. // Generate a new random bucket name.
  1323. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  1324. args["bucketName"] = bucketName
  1325. // Make a new bucket.
  1326. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
  1327. if err != nil {
  1328. logError(testName, function, args, startTime, "", "Make bucket failed", err)
  1329. return
  1330. }
  1331. err = c.EnableVersioning(context.Background(), bucketName)
  1332. if err != nil {
  1333. logError(testName, function, args, startTime, "", "Enable versioning failed", err)
  1334. return
  1335. }
  1336. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  1337. args["objectName"] = objectName
  1338. // var testFiles = []string{"datafile-5-MB", "datafile-10-kB"}
  1339. var testFiles = []string{"datafile-5-MB", "datafile-10-kB"}
  1340. var testFilesBytes [][]byte
  1341. for _, testFile := range testFiles {
  1342. r := getDataReader(testFile)
  1343. buf, err := ioutil.ReadAll(r)
  1344. if err != nil {
  1345. logError(testName, function, args, startTime, "", "unexpected failure", err)
  1346. return
  1347. }
  1348. r.Close()
  1349. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{})
  1350. if err != nil {
  1351. logError(testName, function, args, startTime, "", "PutObject failed", err)
  1352. return
  1353. }
  1354. testFilesBytes = append(testFilesBytes, buf)
  1355. }
  1356. objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
  1357. var results []minio.ObjectInfo
  1358. for info := range objectsInfo {
  1359. if info.Err != nil {
  1360. logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
  1361. return
  1362. }
  1363. results = append(results, info)
  1364. }
  1365. sort.SliceStable(results, func(i, j int) bool {
  1366. return results[i].Size > results[j].Size
  1367. })
  1368. // Source objects to concatenate. We also specify decryption
  1369. // key for each
  1370. src1 := minio.CopySrcOptions{
  1371. Bucket: bucketName,
  1372. Object: objectName,
  1373. VersionID: results[0].VersionID,
  1374. }
  1375. src2 := minio.CopySrcOptions{
  1376. Bucket: bucketName,
  1377. Object: objectName,
  1378. VersionID: results[1].VersionID,
  1379. }
  1380. dst := minio.CopyDestOptions{
  1381. Bucket: bucketName,
  1382. Object: objectName + "-copy",
  1383. }
  1384. _, err = c.ComposeObject(context.Background(), dst, src1, src2)
  1385. if err != nil {
  1386. logError(testName, function, args, startTime, "", "ComposeObject failed", err)
  1387. return
  1388. }
  1389. // Destination object
  1390. readerCopy, err := c.GetObject(context.Background(), bucketName, objectName+"-copy", minio.GetObjectOptions{})
  1391. if err != nil {
  1392. logError(testName, function, args, startTime, "", "GetObject of the copy object failed", err)
  1393. return
  1394. }
  1395. defer readerCopy.Close()
  1396. copyContentBytes, err := ioutil.ReadAll(readerCopy)
  1397. if err != nil {
  1398. logError(testName, function, args, startTime, "", "Reading from the copy object reader failed", err)
  1399. return
  1400. }
  1401. var expectedContent []byte
  1402. for _, fileBytes := range testFilesBytes {
  1403. expectedContent = append(expectedContent, fileBytes...)
  1404. }
  1405. if len(copyContentBytes) == 0 || !bytes.Equal(copyContentBytes, expectedContent) {
  1406. logError(testName, function, args, startTime, "", "Unexpected destination object content", err)
  1407. return
  1408. }
  1409. // Delete all objects and their versions as long as the bucket itself
  1410. if err = cleanupVersionedBucket(bucketName, c); err != nil {
  1411. logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
  1412. return
  1413. }
  1414. successLogger(testName, function, args, startTime).Info()
  1415. }
  1416. func testRemoveObjectWithVersioning() {
  1417. // initialize logging params
  1418. startTime := time.Now()
  1419. testName := getFuncName()
  1420. function := "DeleteObject()"
  1421. args := map[string]interface{}{}
  1422. // Seed random based on current time.
  1423. rand.Seed(time.Now().Unix())
  1424. // Instantiate new minio client object.
  1425. c, err := minio.New(os.Getenv(serverEndpoint),
  1426. &minio.Options{
  1427. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  1428. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  1429. })
  1430. if err != nil {
  1431. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  1432. return
  1433. }
  1434. // Enable tracing, write to stderr.
  1435. // c.TraceOn(os.Stderr)
  1436. // Set user agent.
  1437. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  1438. // Generate a new random bucket name.
  1439. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  1440. args["bucketName"] = bucketName
  1441. // Make a new bucket.
  1442. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
  1443. if err != nil {
  1444. logError(testName, function, args, startTime, "", "Make bucket failed", err)
  1445. return
  1446. }
  1447. err = c.EnableVersioning(context.Background(), bucketName)
  1448. if err != nil {
  1449. logError(testName, function, args, startTime, "", "Enable versioning failed", err)
  1450. return
  1451. }
  1452. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  1453. args["objectName"] = objectName
  1454. _, err = c.PutObject(context.Background(), bucketName, objectName, getDataReader("datafile-10-kB"), int64(dataFileMap["datafile-10-kB"]), minio.PutObjectOptions{})
  1455. if err != nil {
  1456. logError(testName, function, args, startTime, "", "PutObject failed", err)
  1457. return
  1458. }
  1459. objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
  1460. var version minio.ObjectInfo
  1461. for info := range objectsInfo {
  1462. if info.Err != nil {
  1463. logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
  1464. return
  1465. }
  1466. version = info
  1467. break
  1468. }
  1469. err = c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{VersionID: version.VersionID})
  1470. if err != nil {
  1471. logError(testName, function, args, startTime, "", "DeleteObject failed", err)
  1472. return
  1473. }
  1474. objectsInfo = c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
  1475. for range objectsInfo {
  1476. logError(testName, function, args, startTime, "", "Unexpected versioning info, should not have any one ", err)
  1477. return
  1478. }
  1479. err = c.RemoveBucket(context.Background(), bucketName)
  1480. if err != nil {
  1481. logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
  1482. return
  1483. }
  1484. successLogger(testName, function, args, startTime).Info()
  1485. }
  1486. func testRemoveObjectsWithVersioning() {
  1487. // initialize logging params
  1488. startTime := time.Now()
  1489. testName := getFuncName()
  1490. function := "DeleteObjects()"
  1491. args := map[string]interface{}{}
  1492. // Seed random based on current time.
  1493. rand.Seed(time.Now().Unix())
  1494. // Instantiate new minio client object.
  1495. c, err := minio.New(os.Getenv(serverEndpoint),
  1496. &minio.Options{
  1497. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  1498. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  1499. })
  1500. if err != nil {
  1501. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  1502. return
  1503. }
  1504. // Enable tracing, write to stderr.
  1505. // c.TraceOn(os.Stderr)
  1506. // Set user agent.
  1507. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  1508. // Generate a new random bucket name.
  1509. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  1510. args["bucketName"] = bucketName
  1511. // Make a new bucket.
  1512. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
  1513. if err != nil {
  1514. logError(testName, function, args, startTime, "", "Make bucket failed", err)
  1515. return
  1516. }
  1517. err = c.EnableVersioning(context.Background(), bucketName)
  1518. if err != nil {
  1519. logError(testName, function, args, startTime, "", "Enable versioning failed", err)
  1520. return
  1521. }
  1522. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  1523. args["objectName"] = objectName
  1524. _, err = c.PutObject(context.Background(), bucketName, objectName, getDataReader("datafile-10-kB"), int64(dataFileMap["datafile-10-kB"]), minio.PutObjectOptions{})
  1525. if err != nil {
  1526. logError(testName, function, args, startTime, "", "PutObject failed", err)
  1527. return
  1528. }
  1529. objectsVersions := make(chan minio.ObjectInfo)
  1530. go func() {
  1531. objectsVersionsInfo := c.ListObjects(context.Background(), bucketName,
  1532. minio.ListObjectsOptions{WithVersions: true, Recursive: true})
  1533. for info := range objectsVersionsInfo {
  1534. if info.Err != nil {
  1535. logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
  1536. return
  1537. }
  1538. objectsVersions <- info
  1539. }
  1540. close(objectsVersions)
  1541. }()
  1542. removeErrors := c.RemoveObjects(context.Background(), bucketName, objectsVersions, minio.RemoveObjectsOptions{})
  1543. if err != nil {
  1544. logError(testName, function, args, startTime, "", "DeleteObjects call failed", err)
  1545. return
  1546. }
  1547. for e := range removeErrors {
  1548. if e.Err != nil {
  1549. logError(testName, function, args, startTime, "", "Single delete operation failed", err)
  1550. return
  1551. }
  1552. }
  1553. objectsVersionsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
  1554. for range objectsVersionsInfo {
  1555. logError(testName, function, args, startTime, "", "Unexpected versioning info, should not have any one ", err)
  1556. return
  1557. }
  1558. err = c.RemoveBucket(context.Background(), bucketName)
  1559. if err != nil {
  1560. logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
  1561. return
  1562. }
  1563. successLogger(testName, function, args, startTime).Info()
  1564. }
  1565. func testObjectTaggingWithVersioning() {
  1566. // initialize logging params
  1567. startTime := time.Now()
  1568. testName := getFuncName()
  1569. function := "{Get,Set,Remove}ObjectTagging()"
  1570. args := map[string]interface{}{}
  1571. // Seed random based on current time.
  1572. rand.Seed(time.Now().Unix())
  1573. // Instantiate new minio client object.
  1574. c, err := minio.New(os.Getenv(serverEndpoint),
  1575. &minio.Options{
  1576. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  1577. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  1578. })
  1579. if err != nil {
  1580. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  1581. return
  1582. }
  1583. // Enable tracing, write to stderr.
  1584. // c.TraceOn(os.Stderr)
  1585. // Set user agent.
  1586. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  1587. // Generate a new random bucket name.
  1588. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  1589. args["bucketName"] = bucketName
  1590. // Make a new bucket.
  1591. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
  1592. if err != nil {
  1593. logError(testName, function, args, startTime, "", "Make bucket failed", err)
  1594. return
  1595. }
  1596. err = c.EnableVersioning(context.Background(), bucketName)
  1597. if err != nil {
  1598. logError(testName, function, args, startTime, "", "Enable versioning failed", err)
  1599. return
  1600. }
  1601. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  1602. args["objectName"] = objectName
  1603. for _, file := range []string{"datafile-1-b", "datafile-10-kB"} {
  1604. _, err = c.PutObject(context.Background(), bucketName, objectName, getDataReader(file), int64(dataFileMap[file]), minio.PutObjectOptions{})
  1605. if err != nil {
  1606. logError(testName, function, args, startTime, "", "PutObject failed", err)
  1607. return
  1608. }
  1609. }
  1610. versionsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
  1611. var versions []minio.ObjectInfo
  1612. for info := range versionsInfo {
  1613. if info.Err != nil {
  1614. logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
  1615. return
  1616. }
  1617. versions = append(versions, info)
  1618. }
  1619. sort.SliceStable(versions, func(i, j int) bool {
  1620. return versions[i].Size < versions[j].Size
  1621. })
  1622. tagsV1 := map[string]string{"key1": "val1"}
  1623. t1, err := tags.MapToObjectTags(tagsV1)
  1624. if err != nil {
  1625. logError(testName, function, args, startTime, "", "PutObjectTagging (1) failed", err)
  1626. return
  1627. }
  1628. err = c.PutObjectTagging(context.Background(), bucketName, objectName, t1, minio.PutObjectTaggingOptions{VersionID: versions[0].VersionID})
  1629. if err != nil {
  1630. logError(testName, function, args, startTime, "", "PutObjectTagging (1) failed", err)
  1631. return
  1632. }
  1633. tagsV2 := map[string]string{"key2": "val2"}
  1634. t2, err := tags.MapToObjectTags(tagsV2)
  1635. if err != nil {
  1636. logError(testName, function, args, startTime, "", "PutObjectTagging (1) failed", err)
  1637. return
  1638. }
  1639. err = c.PutObjectTagging(context.Background(), bucketName, objectName, t2, minio.PutObjectTaggingOptions{VersionID: versions[1].VersionID})
  1640. if err != nil {
  1641. logError(testName, function, args, startTime, "", "PutObjectTagging (2) failed", err)
  1642. return
  1643. }
  1644. tagsEqual := func(tags1, tags2 map[string]string) bool {
  1645. for k1, v1 := range tags1 {
  1646. v2, found := tags2[k1]
  1647. if found {
  1648. if v1 != v2 {
  1649. return false
  1650. }
  1651. }
  1652. }
  1653. return true
  1654. }
  1655. gotTagsV1, err := c.GetObjectTagging(context.Background(), bucketName, objectName, minio.GetObjectTaggingOptions{VersionID: versions[0].VersionID})
  1656. if err != nil {
  1657. logError(testName, function, args, startTime, "", "GetObjectTagging failed", err)
  1658. return
  1659. }
  1660. if !tagsEqual(t1.ToMap(), gotTagsV1.ToMap()) {
  1661. logError(testName, function, args, startTime, "", "Unexpected tags content (1)", err)
  1662. return
  1663. }
  1664. gotTagsV2, err := c.GetObjectTagging(context.Background(), bucketName, objectName, minio.GetObjectTaggingOptions{})
  1665. if err != nil {
  1666. logError(testName, function, args, startTime, "", "GetObjectTaggingContext failed", err)
  1667. return
  1668. }
  1669. if !tagsEqual(t2.ToMap(), gotTagsV2.ToMap()) {
  1670. logError(testName, function, args, startTime, "", "Unexpected tags content (2)", err)
  1671. return
  1672. }
  1673. err = c.RemoveObjectTagging(context.Background(), bucketName, objectName, minio.RemoveObjectTaggingOptions{VersionID: versions[0].VersionID})
  1674. if err != nil {
  1675. logError(testName, function, args, startTime, "", "PutObjectTagging (2) failed", err)
  1676. return
  1677. }
  1678. emptyTags, err := c.GetObjectTagging(context.Background(), bucketName, objectName,
  1679. minio.GetObjectTaggingOptions{VersionID: versions[0].VersionID})
  1680. if err != nil {
  1681. logError(testName, function, args, startTime, "", "GetObjectTagging failed", err)
  1682. return
  1683. }
  1684. if len(emptyTags.ToMap()) != 0 {
  1685. logError(testName, function, args, startTime, "", "Unexpected tags content (2)", err)
  1686. return
  1687. }
  1688. // Delete all objects and their versions as long as the bucket itself
  1689. if err = cleanupVersionedBucket(bucketName, c); err != nil {
  1690. logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
  1691. return
  1692. }
  1693. successLogger(testName, function, args, startTime).Info()
  1694. }
  1695. // Test PutObject using a large data to trigger multipart readat
  1696. func testPutObjectWithMetadata() {
  1697. // initialize logging params
  1698. startTime := time.Now()
  1699. testName := getFuncName()
  1700. function := "PutObject(bucketName, objectName, reader,size, opts)"
  1701. args := map[string]interface{}{
  1702. "bucketName": "",
  1703. "objectName": "",
  1704. "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}",
  1705. }
  1706. if !isFullMode() {
  1707. ignoredLog(testName, function, args, startTime, "Skipping functional tests for short/quick runs").Info()
  1708. return
  1709. }
  1710. // Seed random based on current time.
  1711. rand.Seed(time.Now().Unix())
  1712. // Instantiate new minio client object.
  1713. c, err := minio.New(os.Getenv(serverEndpoint),
  1714. &minio.Options{
  1715. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  1716. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  1717. })
  1718. if err != nil {
  1719. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  1720. return
  1721. }
  1722. // Enable tracing, write to stderr.
  1723. // c.TraceOn(os.Stderr)
  1724. // Set user agent.
  1725. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  1726. // Generate a new random bucket name.
  1727. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  1728. args["bucketName"] = bucketName
  1729. // Make a new bucket.
  1730. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  1731. if err != nil {
  1732. logError(testName, function, args, startTime, "", "Make bucket failed", err)
  1733. return
  1734. }
  1735. defer cleanupBucket(bucketName, c)
  1736. bufSize := dataFileMap["datafile-129-MB"]
  1737. var reader = getDataReader("datafile-129-MB")
  1738. defer reader.Close()
  1739. // Save the data
  1740. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  1741. args["objectName"] = objectName
  1742. // Object custom metadata
  1743. customContentType := "custom/contenttype"
  1744. args["metadata"] = map[string][]string{
  1745. "Content-Type": {customContentType},
  1746. "X-Amz-Meta-CustomKey": {"extra spaces in value"},
  1747. }
  1748. _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{
  1749. ContentType: customContentType})
  1750. if err != nil {
  1751. logError(testName, function, args, startTime, "", "PutObject failed", err)
  1752. return
  1753. }
  1754. // Read the data back
  1755. r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  1756. if err != nil {
  1757. logError(testName, function, args, startTime, "", "GetObject failed", err)
  1758. return
  1759. }
  1760. st, err := r.Stat()
  1761. if err != nil {
  1762. logError(testName, function, args, startTime, "", "Stat failed", err)
  1763. return
  1764. }
  1765. if st.Size != int64(bufSize) {
  1766. logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err)
  1767. return
  1768. }
  1769. if st.ContentType != customContentType && st.ContentType != "application/octet-stream" {
  1770. logError(testName, function, args, startTime, "", "ContentType does not match, expected "+customContentType+" got "+st.ContentType, err)
  1771. return
  1772. }
  1773. if err := crcMatchesName(r, "datafile-129-MB"); err != nil {
  1774. logError(testName, function, args, startTime, "", "data CRC check failed", err)
  1775. return
  1776. }
  1777. if err := r.Close(); err != nil {
  1778. logError(testName, function, args, startTime, "", "Object Close failed", err)
  1779. return
  1780. }
  1781. if err := r.Close(); err == nil {
  1782. logError(testName, function, args, startTime, "", "Object already closed, should respond with error", err)
  1783. return
  1784. }
  1785. successLogger(testName, function, args, startTime).Info()
  1786. }
  1787. func testPutObjectWithContentLanguage() {
  1788. // initialize logging params
  1789. objectName := "test-object"
  1790. startTime := time.Now()
  1791. testName := getFuncName()
  1792. function := "PutObject(bucketName, objectName, reader, size, opts)"
  1793. args := map[string]interface{}{
  1794. "bucketName": "",
  1795. "objectName": objectName,
  1796. "size": -1,
  1797. "opts": "",
  1798. }
  1799. // Seed random based on current time.
  1800. rand.Seed(time.Now().Unix())
  1801. // Instantiate new minio client object.
  1802. c, err := minio.New(os.Getenv(serverEndpoint),
  1803. &minio.Options{
  1804. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  1805. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  1806. })
  1807. if err != nil {
  1808. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  1809. return
  1810. }
  1811. // Enable tracing, write to stderr.
  1812. // c.TraceOn(os.Stderr)
  1813. // Set user agent.
  1814. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  1815. // Generate a new random bucket name.
  1816. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  1817. args["bucketName"] = bucketName
  1818. // Make a new bucket.
  1819. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  1820. if err != nil {
  1821. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  1822. return
  1823. }
  1824. defer cleanupBucket(bucketName, c)
  1825. data := []byte{}
  1826. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(data), int64(0), minio.PutObjectOptions{
  1827. ContentLanguage: "en",
  1828. })
  1829. if err != nil {
  1830. logError(testName, function, args, startTime, "", "PutObject failed", err)
  1831. return
  1832. }
  1833. objInfo, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
  1834. if err != nil {
  1835. logError(testName, function, args, startTime, "", "StatObject failed", err)
  1836. return
  1837. }
  1838. if objInfo.Metadata.Get("Content-Language") != "en" {
  1839. logError(testName, function, args, startTime, "", "Expected content-language 'en' doesn't match with StatObject return value", err)
  1840. return
  1841. }
  1842. successLogger(testName, function, args, startTime).Info()
  1843. }
  1844. // Test put object with streaming signature.
  1845. func testPutObjectStreaming() {
  1846. // initialize logging params
  1847. objectName := "test-object"
  1848. startTime := time.Now()
  1849. testName := getFuncName()
  1850. function := "PutObject(bucketName, objectName, reader,size,opts)"
  1851. args := map[string]interface{}{
  1852. "bucketName": "",
  1853. "objectName": objectName,
  1854. "size": -1,
  1855. "opts": "",
  1856. }
  1857. // Seed random based on current time.
  1858. rand.Seed(time.Now().Unix())
  1859. // Instantiate new minio client object.
  1860. c, err := minio.New(os.Getenv(serverEndpoint),
  1861. &minio.Options{
  1862. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  1863. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  1864. })
  1865. if err != nil {
  1866. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  1867. return
  1868. }
  1869. // Enable tracing, write to stderr.
  1870. // c.TraceOn(os.Stderr)
  1871. // Set user agent.
  1872. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  1873. // Generate a new random bucket name.
  1874. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  1875. args["bucketName"] = bucketName
  1876. // Make a new bucket.
  1877. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  1878. if err != nil {
  1879. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  1880. return
  1881. }
  1882. defer cleanupBucket(bucketName, c)
  1883. // Upload an object.
  1884. sizes := []int64{0, 64*1024 - 1, 64 * 1024}
  1885. for _, size := range sizes {
  1886. data := newRandomReader(size, size)
  1887. ui, err := c.PutObject(context.Background(), bucketName, objectName, data, int64(size), minio.PutObjectOptions{})
  1888. if err != nil {
  1889. logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err)
  1890. return
  1891. }
  1892. if ui.Size != size {
  1893. logError(testName, function, args, startTime, "", "PutObjectStreaming result has unexpected size", nil)
  1894. return
  1895. }
  1896. objInfo, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
  1897. if err != nil {
  1898. logError(testName, function, args, startTime, "", "StatObject failed", err)
  1899. return
  1900. }
  1901. if objInfo.Size != size {
  1902. logError(testName, function, args, startTime, "", "Unexpected size", err)
  1903. return
  1904. }
  1905. }
  1906. successLogger(testName, function, args, startTime).Info()
  1907. }
  1908. // Test get object seeker from the end, using whence set to '2'.
  1909. func testGetObjectSeekEnd() {
  1910. // initialize logging params
  1911. startTime := time.Now()
  1912. testName := getFuncName()
  1913. function := "GetObject(bucketName, objectName)"
  1914. args := map[string]interface{}{}
  1915. // Seed random based on current time.
  1916. rand.Seed(time.Now().Unix())
  1917. // Instantiate new minio client object.
  1918. c, err := minio.New(os.Getenv(serverEndpoint),
  1919. &minio.Options{
  1920. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  1921. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  1922. })
  1923. if err != nil {
  1924. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  1925. return
  1926. }
  1927. // Enable tracing, write to stderr.
  1928. // c.TraceOn(os.Stderr)
  1929. // Set user agent.
  1930. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  1931. // Generate a new random bucket name.
  1932. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  1933. args["bucketName"] = bucketName
  1934. // Make a new bucket.
  1935. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  1936. if err != nil {
  1937. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  1938. return
  1939. }
  1940. defer cleanupBucket(bucketName, c)
  1941. // Generate 33K of data.
  1942. bufSize := dataFileMap["datafile-33-kB"]
  1943. var reader = getDataReader("datafile-33-kB")
  1944. defer reader.Close()
  1945. // Save the data
  1946. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  1947. args["objectName"] = objectName
  1948. buf, err := ioutil.ReadAll(reader)
  1949. if err != nil {
  1950. logError(testName, function, args, startTime, "", "ReadAll failed", err)
  1951. return
  1952. }
  1953. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  1954. if err != nil {
  1955. logError(testName, function, args, startTime, "", "PutObject failed", err)
  1956. return
  1957. }
  1958. // Read the data back
  1959. r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  1960. if err != nil {
  1961. logError(testName, function, args, startTime, "", "GetObject failed", err)
  1962. return
  1963. }
  1964. st, err := r.Stat()
  1965. if err != nil {
  1966. logError(testName, function, args, startTime, "", "Stat failed", err)
  1967. return
  1968. }
  1969. if st.Size != int64(bufSize) {
  1970. logError(testName, function, args, startTime, "", "Number of bytes read does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err)
  1971. return
  1972. }
  1973. pos, err := r.Seek(-100, 2)
  1974. if err != nil {
  1975. logError(testName, function, args, startTime, "", "Object Seek failed", err)
  1976. return
  1977. }
  1978. if pos != st.Size-100 {
  1979. logError(testName, function, args, startTime, "", "Incorrect position", err)
  1980. return
  1981. }
  1982. buf2 := make([]byte, 100)
  1983. m, err := readFull(r, buf2)
  1984. if err != nil {
  1985. logError(testName, function, args, startTime, "", "Error reading through readFull", err)
  1986. return
  1987. }
  1988. if m != len(buf2) {
  1989. logError(testName, function, args, startTime, "", "Number of bytes dont match, expected "+string(len(buf2))+" got "+string(m), err)
  1990. return
  1991. }
  1992. hexBuf1 := fmt.Sprintf("%02x", buf[len(buf)-100:])
  1993. hexBuf2 := fmt.Sprintf("%02x", buf2[:m])
  1994. if hexBuf1 != hexBuf2 {
  1995. logError(testName, function, args, startTime, "", "Values at same index dont match", err)
  1996. return
  1997. }
  1998. pos, err = r.Seek(-100, 2)
  1999. if err != nil {
  2000. logError(testName, function, args, startTime, "", "Object Seek failed", err)
  2001. return
  2002. }
  2003. if pos != st.Size-100 {
  2004. logError(testName, function, args, startTime, "", "Incorrect position", err)
  2005. return
  2006. }
  2007. if err = r.Close(); err != nil {
  2008. logError(testName, function, args, startTime, "", "ObjectClose failed", err)
  2009. return
  2010. }
  2011. successLogger(testName, function, args, startTime).Info()
  2012. }
  2013. // Test get object reader to not throw error on being closed twice.
  2014. func testGetObjectClosedTwice() {
  2015. // initialize logging params
  2016. startTime := time.Now()
  2017. testName := getFuncName()
  2018. function := "GetObject(bucketName, objectName)"
  2019. args := map[string]interface{}{}
  2020. // Seed random based on current time.
  2021. rand.Seed(time.Now().Unix())
  2022. // Instantiate new minio client object.
  2023. c, err := minio.New(os.Getenv(serverEndpoint),
  2024. &minio.Options{
  2025. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  2026. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  2027. })
  2028. if err != nil {
  2029. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  2030. return
  2031. }
  2032. // Enable tracing, write to stderr.
  2033. // c.TraceOn(os.Stderr)
  2034. // Set user agent.
  2035. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  2036. // Generate a new random bucket name.
  2037. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  2038. args["bucketName"] = bucketName
  2039. // Make a new bucket.
  2040. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  2041. if err != nil {
  2042. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  2043. return
  2044. }
  2045. defer cleanupBucket(bucketName, c)
  2046. // Generate 33K of data.
  2047. bufSize := dataFileMap["datafile-33-kB"]
  2048. var reader = getDataReader("datafile-33-kB")
  2049. defer reader.Close()
  2050. // Save the data
  2051. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  2052. args["objectName"] = objectName
  2053. _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  2054. if err != nil {
  2055. logError(testName, function, args, startTime, "", "PutObject failed", err)
  2056. return
  2057. }
  2058. // Read the data back
  2059. r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  2060. if err != nil {
  2061. logError(testName, function, args, startTime, "", "GetObject failed", err)
  2062. return
  2063. }
  2064. st, err := r.Stat()
  2065. if err != nil {
  2066. logError(testName, function, args, startTime, "", "Stat failed", err)
  2067. return
  2068. }
  2069. if st.Size != int64(bufSize) {
  2070. logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err)
  2071. return
  2072. }
  2073. if err := crcMatchesName(r, "datafile-33-kB"); err != nil {
  2074. logError(testName, function, args, startTime, "", "data CRC check failed", err)
  2075. return
  2076. }
  2077. if err := r.Close(); err != nil {
  2078. logError(testName, function, args, startTime, "", "Object Close failed", err)
  2079. return
  2080. }
  2081. if err := r.Close(); err == nil {
  2082. logError(testName, function, args, startTime, "", "Already closed object. No error returned", err)
  2083. return
  2084. }
  2085. successLogger(testName, function, args, startTime).Info()
  2086. }
  2087. // Test RemoveObjects request where context cancels after timeout
  2088. func testRemoveObjectsContext() {
  2089. // Initialize logging params.
  2090. startTime := time.Now()
  2091. testName := getFuncName()
  2092. function := "RemoveObjects(ctx, bucketName, objectsCh)"
  2093. args := map[string]interface{}{
  2094. "bucketName": "",
  2095. }
  2096. // Seed random based on current tie.
  2097. rand.Seed(time.Now().Unix())
  2098. // Instantiate new minio client.
  2099. c, err := minio.New(os.Getenv(serverEndpoint),
  2100. &minio.Options{
  2101. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  2102. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  2103. })
  2104. if err != nil {
  2105. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  2106. return
  2107. }
  2108. // Set user agent.
  2109. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  2110. // Enable tracing, write to stdout.
  2111. // c.TraceOn(os.Stderr)
  2112. // Generate a new random bucket name.
  2113. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  2114. args["bucketName"] = bucketName
  2115. // Make a new bucket.
  2116. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  2117. if err != nil {
  2118. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  2119. return
  2120. }
  2121. defer cleanupBucket(bucketName, c)
  2122. // Generate put data.
  2123. r := bytes.NewReader(bytes.Repeat([]byte("a"), 8))
  2124. // Multi remove of 20 objects.
  2125. nrObjects := 20
  2126. objectsCh := make(chan minio.ObjectInfo)
  2127. go func() {
  2128. defer close(objectsCh)
  2129. for i := 0; i < nrObjects; i++ {
  2130. objectName := "sample" + strconv.Itoa(i) + ".txt"
  2131. info, err := c.PutObject(context.Background(), bucketName, objectName, r, 8,
  2132. minio.PutObjectOptions{ContentType: "application/octet-stream"})
  2133. if err != nil {
  2134. logError(testName, function, args, startTime, "", "PutObject failed", err)
  2135. continue
  2136. }
  2137. objectsCh <- minio.ObjectInfo{
  2138. Key: info.Key,
  2139. VersionID: info.VersionID,
  2140. }
  2141. }
  2142. }()
  2143. // Set context to cancel in 1 nanosecond.
  2144. ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
  2145. args["ctx"] = ctx
  2146. defer cancel()
  2147. // Call RemoveObjects API with short timeout.
  2148. errorCh := c.RemoveObjects(ctx, bucketName, objectsCh, minio.RemoveObjectsOptions{})
  2149. // Check for error.
  2150. select {
  2151. case r := <-errorCh:
  2152. if r.Err == nil {
  2153. logError(testName, function, args, startTime, "", "RemoveObjects should fail on short timeout", err)
  2154. return
  2155. }
  2156. }
  2157. // Set context with longer timeout.
  2158. ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
  2159. args["ctx"] = ctx
  2160. defer cancel()
  2161. // Perform RemoveObjects with the longer timeout. Expect the removals to succeed.
  2162. errorCh = c.RemoveObjects(ctx, bucketName, objectsCh, minio.RemoveObjectsOptions{})
  2163. select {
  2164. case r, more := <-errorCh:
  2165. if more || r.Err != nil {
  2166. logError(testName, function, args, startTime, "", "Unexpected error", r.Err)
  2167. return
  2168. }
  2169. }
  2170. successLogger(testName, function, args, startTime).Info()
  2171. }
  2172. // Test removing multiple objects with Remove API
  2173. func testRemoveMultipleObjects() {
  2174. // initialize logging params
  2175. startTime := time.Now()
  2176. testName := getFuncName()
  2177. function := "RemoveObjects(bucketName, objectsCh)"
  2178. args := map[string]interface{}{
  2179. "bucketName": "",
  2180. }
  2181. // Seed random based on current time.
  2182. rand.Seed(time.Now().Unix())
  2183. // Instantiate new minio client object.
  2184. c, err := minio.New(os.Getenv(serverEndpoint),
  2185. &minio.Options{
  2186. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  2187. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  2188. })
  2189. if err != nil {
  2190. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  2191. return
  2192. }
  2193. // Set user agent.
  2194. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  2195. // Enable tracing, write to stdout.
  2196. // c.TraceOn(os.Stderr)
  2197. // Generate a new random bucket name.
  2198. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  2199. args["bucketName"] = bucketName
  2200. // Make a new bucket.
  2201. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  2202. if err != nil {
  2203. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  2204. return
  2205. }
  2206. defer cleanupBucket(bucketName, c)
  2207. r := bytes.NewReader(bytes.Repeat([]byte("a"), 8))
  2208. // Multi remove of 1100 objects
  2209. nrObjects := 200
  2210. objectsCh := make(chan minio.ObjectInfo)
  2211. go func() {
  2212. defer close(objectsCh)
  2213. // Upload objects and send them to objectsCh
  2214. for i := 0; i < nrObjects; i++ {
  2215. objectName := "sample" + strconv.Itoa(i) + ".txt"
  2216. info, err := c.PutObject(context.Background(), bucketName, objectName, r, 8,
  2217. minio.PutObjectOptions{ContentType: "application/octet-stream"})
  2218. if err != nil {
  2219. logError(testName, function, args, startTime, "", "PutObject failed", err)
  2220. continue
  2221. }
  2222. objectsCh <- minio.ObjectInfo{
  2223. Key: info.Key,
  2224. VersionID: info.VersionID,
  2225. }
  2226. }
  2227. }()
  2228. // Call RemoveObjects API
  2229. errorCh := c.RemoveObjects(context.Background(), bucketName, objectsCh, minio.RemoveObjectsOptions{})
  2230. // Check if errorCh doesn't receive any error
  2231. select {
  2232. case r, more := <-errorCh:
  2233. if more {
  2234. logError(testName, function, args, startTime, "", "Unexpected error", r.Err)
  2235. return
  2236. }
  2237. }
  2238. successLogger(testName, function, args, startTime).Info()
  2239. }
  2240. // Tests FPutObject of a big file to trigger multipart
  2241. func testFPutObjectMultipart() {
  2242. // initialize logging params
  2243. startTime := time.Now()
  2244. testName := getFuncName()
  2245. function := "FPutObject(bucketName, objectName, fileName, opts)"
  2246. args := map[string]interface{}{
  2247. "bucketName": "",
  2248. "objectName": "",
  2249. "fileName": "",
  2250. "opts": "",
  2251. }
  2252. // Seed random based on current time.
  2253. rand.Seed(time.Now().Unix())
  2254. // Instantiate new minio client object.
  2255. c, err := minio.New(os.Getenv(serverEndpoint),
  2256. &minio.Options{
  2257. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  2258. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  2259. })
  2260. if err != nil {
  2261. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  2262. return
  2263. }
  2264. // Enable tracing, write to stderr.
  2265. // c.TraceOn(os.Stderr)
  2266. // Set user agent.
  2267. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  2268. // Generate a new random bucket name.
  2269. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  2270. args["bucketName"] = bucketName
  2271. // Make a new bucket.
  2272. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  2273. if err != nil {
  2274. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  2275. return
  2276. }
  2277. defer cleanupBucket(bucketName, c)
  2278. // Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload.
  2279. var fileName = getMintDataDirFilePath("datafile-129-MB")
  2280. if fileName == "" {
  2281. // Make a temp file with minPartSize bytes of data.
  2282. file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest")
  2283. if err != nil {
  2284. logError(testName, function, args, startTime, "", "TempFile creation failed", err)
  2285. return
  2286. }
  2287. // Upload 2 parts to utilize all 3 'workers' in multipart and still have a part to upload.
  2288. if _, err = io.Copy(file, getDataReader("datafile-129-MB")); err != nil {
  2289. logError(testName, function, args, startTime, "", "Copy failed", err)
  2290. return
  2291. }
  2292. if err = file.Close(); err != nil {
  2293. logError(testName, function, args, startTime, "", "File Close failed", err)
  2294. return
  2295. }
  2296. fileName = file.Name()
  2297. args["fileName"] = fileName
  2298. }
  2299. totalSize := dataFileMap["datafile-129-MB"]
  2300. // Set base object name
  2301. objectName := bucketName + "FPutObject" + "-standard"
  2302. args["objectName"] = objectName
  2303. objectContentType := "testapplication/octet-stream"
  2304. args["objectContentType"] = objectContentType
  2305. // Perform standard FPutObject with contentType provided (Expecting application/octet-stream)
  2306. _, err = c.FPutObject(context.Background(), bucketName, objectName, fileName, minio.PutObjectOptions{ContentType: objectContentType})
  2307. if err != nil {
  2308. logError(testName, function, args, startTime, "", "FPutObject failed", err)
  2309. return
  2310. }
  2311. r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  2312. if err != nil {
  2313. logError(testName, function, args, startTime, "", "GetObject failed", err)
  2314. return
  2315. }
  2316. objInfo, err := r.Stat()
  2317. if err != nil {
  2318. logError(testName, function, args, startTime, "", "Unexpected error", err)
  2319. return
  2320. }
  2321. if objInfo.Size != int64(totalSize) {
  2322. logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(totalSize))+" got "+string(objInfo.Size), err)
  2323. return
  2324. }
  2325. if objInfo.ContentType != objectContentType && objInfo.ContentType != "application/octet-stream" {
  2326. logError(testName, function, args, startTime, "", "ContentType doesn't match", err)
  2327. return
  2328. }
  2329. successLogger(testName, function, args, startTime).Info()
  2330. }
  2331. // Tests FPutObject with null contentType (default = application/octet-stream)
  2332. func testFPutObject() {
  2333. // initialize logging params
  2334. startTime := time.Now()
  2335. testName := getFuncName()
  2336. function := "FPutObject(bucketName, objectName, fileName, opts)"
  2337. args := map[string]interface{}{
  2338. "bucketName": "",
  2339. "objectName": "",
  2340. "fileName": "",
  2341. "opts": "",
  2342. }
  2343. // Seed random based on current time.
  2344. rand.Seed(time.Now().Unix())
  2345. // Instantiate new minio client object.
  2346. c, err := minio.New(os.Getenv(serverEndpoint),
  2347. &minio.Options{
  2348. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  2349. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  2350. })
  2351. if err != nil {
  2352. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  2353. return
  2354. }
  2355. // Enable tracing, write to stderr.
  2356. // c.TraceOn(os.Stderr)
  2357. // Set user agent.
  2358. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  2359. // Generate a new random bucket name.
  2360. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  2361. location := "us-east-1"
  2362. // Make a new bucket.
  2363. args["bucketName"] = bucketName
  2364. args["location"] = location
  2365. function = "MakeBucket(bucketName, location)"
  2366. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: location})
  2367. if err != nil {
  2368. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  2369. return
  2370. }
  2371. defer cleanupBucket(bucketName, c)
  2372. // Upload 3 parts worth of data to use all 3 of multiparts 'workers' and have an extra part.
  2373. // Use different data in part for multipart tests to check parts are uploaded in correct order.
  2374. var fName = getMintDataDirFilePath("datafile-129-MB")
  2375. if fName == "" {
  2376. // Make a temp file with minPartSize bytes of data.
  2377. file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest")
  2378. if err != nil {
  2379. logError(testName, function, args, startTime, "", "TempFile creation failed", err)
  2380. return
  2381. }
  2382. // Upload 3 parts to utilize all 3 'workers' in multipart and still have a part to upload.
  2383. if _, err = io.Copy(file, getDataReader("datafile-129-MB")); err != nil {
  2384. logError(testName, function, args, startTime, "", "File copy failed", err)
  2385. return
  2386. }
  2387. // Close the file pro-actively for windows.
  2388. if err = file.Close(); err != nil {
  2389. logError(testName, function, args, startTime, "", "File close failed", err)
  2390. return
  2391. }
  2392. defer os.Remove(file.Name())
  2393. fName = file.Name()
  2394. }
  2395. // Set base object name
  2396. function = "FPutObject(bucketName, objectName, fileName, opts)"
  2397. objectName := bucketName + "FPutObject"
  2398. args["objectName"] = objectName + "-standard"
  2399. args["fileName"] = fName
  2400. args["opts"] = minio.PutObjectOptions{ContentType: "application/octet-stream"}
  2401. // Perform standard FPutObject with contentType provided (Expecting application/octet-stream)
  2402. ui, err := c.FPutObject(context.Background(), bucketName, objectName+"-standard", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"})
  2403. if err != nil {
  2404. logError(testName, function, args, startTime, "", "FPutObject failed", err)
  2405. return
  2406. }
  2407. if ui.Size != int64(dataFileMap["datafile-129-MB"]) {
  2408. logError(testName, function, args, startTime, "", "FPutObject returned an unexpected upload size", err)
  2409. return
  2410. }
  2411. // Perform FPutObject with no contentType provided (Expecting application/octet-stream)
  2412. args["objectName"] = objectName + "-Octet"
  2413. _, err = c.FPutObject(context.Background(), bucketName, objectName+"-Octet", fName, minio.PutObjectOptions{})
  2414. if err != nil {
  2415. logError(testName, function, args, startTime, "", "File close failed", err)
  2416. return
  2417. }
  2418. srcFile, err := os.Open(fName)
  2419. if err != nil {
  2420. logError(testName, function, args, startTime, "", "File open failed", err)
  2421. return
  2422. }
  2423. defer srcFile.Close()
  2424. // Add extension to temp file name
  2425. tmpFile, err := os.Create(fName + ".gtar")
  2426. if err != nil {
  2427. logError(testName, function, args, startTime, "", "File create failed", err)
  2428. return
  2429. }
  2430. _, err = io.Copy(tmpFile, srcFile)
  2431. if err != nil {
  2432. logError(testName, function, args, startTime, "", "File copy failed", err)
  2433. return
  2434. }
  2435. tmpFile.Close()
  2436. // Perform FPutObject with no contentType provided (Expecting application/x-gtar)
  2437. args["objectName"] = objectName + "-GTar"
  2438. args["opts"] = minio.PutObjectOptions{}
  2439. _, err = c.FPutObject(context.Background(), bucketName, objectName+"-GTar", fName+".gtar", minio.PutObjectOptions{})
  2440. if err != nil {
  2441. logError(testName, function, args, startTime, "", "FPutObject failed", err)
  2442. return
  2443. }
  2444. // Check headers
  2445. function = "StatObject(bucketName, objectName, opts)"
  2446. args["objectName"] = objectName + "-standard"
  2447. rStandard, err := c.StatObject(context.Background(), bucketName, objectName+"-standard", minio.StatObjectOptions{})
  2448. if err != nil {
  2449. logError(testName, function, args, startTime, "", "StatObject failed", err)
  2450. return
  2451. }
  2452. if rStandard.ContentType != "application/octet-stream" {
  2453. logError(testName, function, args, startTime, "", "ContentType does not match, expected application/octet-stream, got "+rStandard.ContentType, err)
  2454. return
  2455. }
  2456. function = "StatObject(bucketName, objectName, opts)"
  2457. args["objectName"] = objectName + "-Octet"
  2458. rOctet, err := c.StatObject(context.Background(), bucketName, objectName+"-Octet", minio.StatObjectOptions{})
  2459. if err != nil {
  2460. logError(testName, function, args, startTime, "", "StatObject failed", err)
  2461. return
  2462. }
  2463. if rOctet.ContentType != "application/octet-stream" {
  2464. logError(testName, function, args, startTime, "", "ContentType does not match, expected application/octet-stream, got "+rOctet.ContentType, err)
  2465. return
  2466. }
  2467. function = "StatObject(bucketName, objectName, opts)"
  2468. args["objectName"] = objectName + "-GTar"
  2469. rGTar, err := c.StatObject(context.Background(), bucketName, objectName+"-GTar", minio.StatObjectOptions{})
  2470. if err != nil {
  2471. logError(testName, function, args, startTime, "", "StatObject failed", err)
  2472. return
  2473. }
  2474. if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" {
  2475. logError(testName, function, args, startTime, "", "ContentType does not match, expected application/x-gtar or application/octet-stream, got "+rGTar.ContentType, err)
  2476. return
  2477. }
  2478. os.Remove(fName + ".gtar")
  2479. successLogger(testName, function, args, startTime).Info()
  2480. }
  2481. // Tests FPutObject request when context cancels after timeout
  2482. func testFPutObjectContext() {
  2483. // initialize logging params
  2484. startTime := time.Now()
  2485. testName := getFuncName()
  2486. function := "FPutObject(bucketName, objectName, fileName, opts)"
  2487. args := map[string]interface{}{
  2488. "bucketName": "",
  2489. "objectName": "",
  2490. "fileName": "",
  2491. "opts": "",
  2492. }
  2493. // Seed random based on current time.
  2494. rand.Seed(time.Now().Unix())
  2495. // Instantiate new minio client object.
  2496. c, err := minio.New(os.Getenv(serverEndpoint),
  2497. &minio.Options{
  2498. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  2499. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  2500. })
  2501. if err != nil {
  2502. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  2503. return
  2504. }
  2505. // Enable tracing, write to stderr.
  2506. // c.TraceOn(os.Stderr)
  2507. // Set user agent.
  2508. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  2509. // Generate a new random bucket name.
  2510. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  2511. args["bucketName"] = bucketName
  2512. // Make a new bucket.
  2513. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  2514. if err != nil {
  2515. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  2516. return
  2517. }
  2518. defer cleanupBucket(bucketName, c)
  2519. // Upload 1 parts worth of data to use multipart upload.
  2520. // Use different data in part for multipart tests to check parts are uploaded in correct order.
  2521. var fName = getMintDataDirFilePath("datafile-1-MB")
  2522. if fName == "" {
  2523. // Make a temp file with 1 MiB bytes of data.
  2524. file, err := ioutil.TempFile(os.TempDir(), "FPutObjectContextTest")
  2525. if err != nil {
  2526. logError(testName, function, args, startTime, "", "TempFile creation failed", err)
  2527. return
  2528. }
  2529. // Upload 1 parts to trigger multipart upload
  2530. if _, err = io.Copy(file, getDataReader("datafile-1-MB")); err != nil {
  2531. logError(testName, function, args, startTime, "", "File copy failed", err)
  2532. return
  2533. }
  2534. // Close the file pro-actively for windows.
  2535. if err = file.Close(); err != nil {
  2536. logError(testName, function, args, startTime, "", "File close failed", err)
  2537. return
  2538. }
  2539. defer os.Remove(file.Name())
  2540. fName = file.Name()
  2541. }
  2542. // Set base object name
  2543. objectName := bucketName + "FPutObjectContext"
  2544. args["objectName"] = objectName
  2545. ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
  2546. args["ctx"] = ctx
  2547. defer cancel()
  2548. // Perform FPutObject with contentType provided (Expecting application/octet-stream)
  2549. _, err = c.FPutObject(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"})
  2550. if err == nil {
  2551. logError(testName, function, args, startTime, "", "FPutObject should fail on short timeout", err)
  2552. return
  2553. }
  2554. ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
  2555. defer cancel()
  2556. // Perform FPutObject with a long timeout. Expect the put object to succeed
  2557. _, err = c.FPutObject(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{})
  2558. if err != nil {
  2559. logError(testName, function, args, startTime, "", "FPutObject shouldn't fail on long timeout", err)
  2560. return
  2561. }
  2562. _, err = c.StatObject(context.Background(), bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{})
  2563. if err != nil {
  2564. logError(testName, function, args, startTime, "", "StatObject failed", err)
  2565. return
  2566. }
  2567. successLogger(testName, function, args, startTime).Info()
  2568. }
  2569. // Tests FPutObject request when context cancels after timeout
  2570. func testFPutObjectContextV2() {
  2571. // initialize logging params
  2572. startTime := time.Now()
  2573. testName := getFuncName()
  2574. function := "FPutObjectContext(ctx, bucketName, objectName, fileName, opts)"
  2575. args := map[string]interface{}{
  2576. "bucketName": "",
  2577. "objectName": "",
  2578. "opts": "minio.PutObjectOptions{ContentType:objectContentType}",
  2579. }
  2580. // Seed random based on current time.
  2581. rand.Seed(time.Now().Unix())
  2582. // Instantiate new minio client object.
  2583. c, err := minio.New(os.Getenv(serverEndpoint),
  2584. &minio.Options{
  2585. Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  2586. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  2587. })
  2588. if err != nil {
  2589. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  2590. return
  2591. }
  2592. // Enable tracing, write to stderr.
  2593. // c.TraceOn(os.Stderr)
  2594. // Set user agent.
  2595. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  2596. // Generate a new random bucket name.
  2597. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  2598. args["bucketName"] = bucketName
  2599. // Make a new bucket.
  2600. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  2601. if err != nil {
  2602. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  2603. return
  2604. }
  2605. defer cleanupBucket(bucketName, c)
  2606. // Upload 1 parts worth of data to use multipart upload.
  2607. // Use different data in part for multipart tests to check parts are uploaded in correct order.
  2608. var fName = getMintDataDirFilePath("datafile-1-MB")
  2609. if fName == "" {
  2610. // Make a temp file with 1 MiB bytes of data.
  2611. file, err := ioutil.TempFile(os.TempDir(), "FPutObjectContextTest")
  2612. if err != nil {
  2613. logError(testName, function, args, startTime, "", "Temp file creation failed", err)
  2614. return
  2615. }
  2616. // Upload 1 parts to trigger multipart upload
  2617. if _, err = io.Copy(file, getDataReader("datafile-1-MB")); err != nil {
  2618. logError(testName, function, args, startTime, "", "File copy failed", err)
  2619. return
  2620. }
  2621. // Close the file pro-actively for windows.
  2622. if err = file.Close(); err != nil {
  2623. logError(testName, function, args, startTime, "", "File close failed", err)
  2624. return
  2625. }
  2626. defer os.Remove(file.Name())
  2627. fName = file.Name()
  2628. }
  2629. // Set base object name
  2630. objectName := bucketName + "FPutObjectContext"
  2631. args["objectName"] = objectName
  2632. ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
  2633. args["ctx"] = ctx
  2634. defer cancel()
  2635. // Perform FPutObject with contentType provided (Expecting application/octet-stream)
  2636. _, err = c.FPutObject(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"})
  2637. if err == nil {
  2638. logError(testName, function, args, startTime, "", "FPutObject should fail on short timeout", err)
  2639. return
  2640. }
  2641. ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
  2642. defer cancel()
  2643. // Perform FPutObject with a long timeout. Expect the put object to succeed
  2644. _, err = c.FPutObject(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{})
  2645. if err != nil {
  2646. logError(testName, function, args, startTime, "", "FPutObject shouldn't fail on longer timeout", err)
  2647. return
  2648. }
  2649. _, err = c.StatObject(context.Background(), bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{})
  2650. if err != nil {
  2651. logError(testName, function, args, startTime, "", "StatObject failed", err)
  2652. return
  2653. }
  2654. successLogger(testName, function, args, startTime).Info()
  2655. }
  2656. // Test validates putObject with context to see if request cancellation is honored.
  2657. func testPutObjectContext() {
  2658. // initialize logging params
  2659. startTime := time.Now()
  2660. testName := getFuncName()
  2661. function := "PutObject(ctx, bucketName, objectName, fileName, opts)"
  2662. args := map[string]interface{}{
  2663. "ctx": "",
  2664. "bucketName": "",
  2665. "objectName": "",
  2666. "opts": "",
  2667. }
  2668. // Instantiate new minio client object.
  2669. c, err := minio.New(os.Getenv(serverEndpoint),
  2670. &minio.Options{
  2671. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  2672. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  2673. })
  2674. if err != nil {
  2675. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  2676. return
  2677. }
  2678. // Enable tracing, write to stderr.
  2679. // c.TraceOn(os.Stderr)
  2680. // Set user agent.
  2681. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  2682. // Make a new bucket.
  2683. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  2684. args["bucketName"] = bucketName
  2685. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  2686. if err != nil {
  2687. logError(testName, function, args, startTime, "", "MakeBucket call failed", err)
  2688. return
  2689. }
  2690. defer cleanupBucket(bucketName, c)
  2691. bufSize := dataFileMap["datafile-33-kB"]
  2692. var reader = getDataReader("datafile-33-kB")
  2693. defer reader.Close()
  2694. objectName := fmt.Sprintf("test-file-%v", rand.Uint32())
  2695. args["objectName"] = objectName
  2696. ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
  2697. cancel()
  2698. args["ctx"] = ctx
  2699. args["opts"] = minio.PutObjectOptions{ContentType: "binary/octet-stream"}
  2700. _, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  2701. if err == nil {
  2702. logError(testName, function, args, startTime, "", "PutObject should fail on short timeout", err)
  2703. return
  2704. }
  2705. ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
  2706. args["ctx"] = ctx
  2707. defer cancel()
  2708. reader = getDataReader("datafile-33-kB")
  2709. defer reader.Close()
  2710. _, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  2711. if err != nil {
  2712. logError(testName, function, args, startTime, "", "PutObject with long timeout failed", err)
  2713. return
  2714. }
  2715. successLogger(testName, function, args, startTime).Info()
  2716. }
  2717. // Tests get object ReaderSeeker interface methods.
  2718. func testGetObjectReadSeekFunctional() {
  2719. // initialize logging params
  2720. startTime := time.Now()
  2721. testName := getFuncName()
  2722. function := "GetObject(bucketName, objectName)"
  2723. args := map[string]interface{}{}
  2724. // Seed random based on current time.
  2725. rand.Seed(time.Now().Unix())
  2726. // Instantiate new minio client object.
  2727. c, err := minio.New(os.Getenv(serverEndpoint),
  2728. &minio.Options{
  2729. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  2730. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  2731. })
  2732. if err != nil {
  2733. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  2734. return
  2735. }
  2736. // Enable tracing, write to stderr.
  2737. // c.TraceOn(os.Stderr)
  2738. // Set user agent.
  2739. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  2740. // Generate a new random bucket name.
  2741. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  2742. args["bucketName"] = bucketName
  2743. // Make a new bucket.
  2744. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  2745. if err != nil {
  2746. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  2747. return
  2748. }
  2749. defer func() {
  2750. // Delete all objects and buckets
  2751. if err = cleanupBucket(bucketName, c); err != nil {
  2752. logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
  2753. return
  2754. }
  2755. }()
  2756. // Generate 33K of data.
  2757. bufSize := dataFileMap["datafile-33-kB"]
  2758. var reader = getDataReader("datafile-33-kB")
  2759. defer reader.Close()
  2760. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  2761. args["objectName"] = objectName
  2762. buf, err := ioutil.ReadAll(reader)
  2763. if err != nil {
  2764. logError(testName, function, args, startTime, "", "ReadAll failed", err)
  2765. return
  2766. }
  2767. // Save the data
  2768. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  2769. if err != nil {
  2770. logError(testName, function, args, startTime, "", "PutObject failed", err)
  2771. return
  2772. }
  2773. // Read the data back
  2774. r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  2775. if err != nil {
  2776. logError(testName, function, args, startTime, "", "GetObject failed", err)
  2777. return
  2778. }
  2779. st, err := r.Stat()
  2780. if err != nil {
  2781. logError(testName, function, args, startTime, "", "Stat object failed", err)
  2782. return
  2783. }
  2784. if st.Size != int64(bufSize) {
  2785. logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err)
  2786. return
  2787. }
  2788. // This following function helps us to compare data from the reader after seek
  2789. // with the data from the original buffer
  2790. cmpData := func(r io.Reader, start, end int) {
  2791. if end-start == 0 {
  2792. return
  2793. }
  2794. buffer := bytes.NewBuffer([]byte{})
  2795. if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil {
  2796. if err != io.EOF {
  2797. logError(testName, function, args, startTime, "", "CopyN failed", err)
  2798. return
  2799. }
  2800. }
  2801. if !bytes.Equal(buf[start:end], buffer.Bytes()) {
  2802. logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err)
  2803. return
  2804. }
  2805. }
  2806. // Generic seek error for errors other than io.EOF
  2807. seekErr := errors.New("seek error")
  2808. testCases := []struct {
  2809. offset int64
  2810. whence int
  2811. pos int64
  2812. err error
  2813. shouldCmp bool
  2814. start int
  2815. end int
  2816. }{
  2817. // Start from offset 0, fetch data and compare
  2818. {0, 0, 0, nil, true, 0, 0},
  2819. // Start from offset 2048, fetch data and compare
  2820. {2048, 0, 2048, nil, true, 2048, bufSize},
  2821. // Start from offset larger than possible
  2822. {int64(bufSize) + 1024, 0, 0, seekErr, false, 0, 0},
  2823. // Move to offset 0 without comparing
  2824. {0, 0, 0, nil, false, 0, 0},
  2825. // Move one step forward and compare
  2826. {1, 1, 1, nil, true, 1, bufSize},
  2827. // Move larger than possible
  2828. {int64(bufSize), 1, 0, seekErr, false, 0, 0},
  2829. // Provide negative offset with CUR_SEEK
  2830. {int64(-1), 1, 0, seekErr, false, 0, 0},
  2831. // Test with whence SEEK_END and with positive offset
  2832. {1024, 2, int64(bufSize) - 1024, io.EOF, true, 0, 0},
  2833. // Test with whence SEEK_END and with negative offset
  2834. {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize},
  2835. // Test with whence SEEK_END and with large negative offset
  2836. {-int64(bufSize) * 2, 2, 0, seekErr, true, 0, 0},
  2837. }
  2838. for i, testCase := range testCases {
  2839. // Perform seek operation
  2840. n, err := r.Seek(testCase.offset, testCase.whence)
  2841. // We expect an error
  2842. if testCase.err == seekErr && err == nil {
  2843. logError(testName, function, args, startTime, "", "Test "+string(i+1)+", unexpected err value: expected: "+testCase.err.Error()+", found: "+err.Error(), err)
  2844. return
  2845. }
  2846. // We expect a specific error
  2847. if testCase.err != seekErr && testCase.err != err {
  2848. logError(testName, function, args, startTime, "", "Test "+string(i+1)+", unexpected err value: expected: "+testCase.err.Error()+", found: "+err.Error(), err)
  2849. return
  2850. }
  2851. // If we expect an error go to the next loop
  2852. if testCase.err != nil {
  2853. continue
  2854. }
  2855. // Check the returned seek pos
  2856. if n != testCase.pos {
  2857. logError(testName, function, args, startTime, "", "Test "+string(i+1)+", number of bytes seeked does not match, expected "+string(testCase.pos)+", got "+string(n), err)
  2858. return
  2859. }
  2860. // Compare only if shouldCmp is activated
  2861. if testCase.shouldCmp {
  2862. cmpData(r, testCase.start, testCase.end)
  2863. }
  2864. }
  2865. successLogger(testName, function, args, startTime).Info()
  2866. }
  2867. // Tests get object ReaderAt interface methods.
  2868. func testGetObjectReadAtFunctional() {
  2869. // initialize logging params
  2870. startTime := time.Now()
  2871. testName := getFuncName()
  2872. function := "GetObject(bucketName, objectName)"
  2873. args := map[string]interface{}{}
  2874. // Seed random based on current time.
  2875. rand.Seed(time.Now().Unix())
  2876. // Instantiate new minio client object.
  2877. c, err := minio.New(os.Getenv(serverEndpoint),
  2878. &minio.Options{
  2879. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  2880. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  2881. })
  2882. if err != nil {
  2883. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  2884. return
  2885. }
  2886. // Enable tracing, write to stderr.
  2887. // c.TraceOn(os.Stderr)
  2888. // Set user agent.
  2889. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  2890. // Generate a new random bucket name.
  2891. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  2892. args["bucketName"] = bucketName
  2893. // Make a new bucket.
  2894. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  2895. if err != nil {
  2896. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  2897. return
  2898. }
  2899. defer cleanupBucket(bucketName, c)
  2900. // Generate 33K of data.
  2901. bufSize := dataFileMap["datafile-33-kB"]
  2902. var reader = getDataReader("datafile-33-kB")
  2903. defer reader.Close()
  2904. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  2905. args["objectName"] = objectName
  2906. buf, err := ioutil.ReadAll(reader)
  2907. if err != nil {
  2908. logError(testName, function, args, startTime, "", "ReadAll failed", err)
  2909. return
  2910. }
  2911. // Save the data
  2912. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  2913. if err != nil {
  2914. logError(testName, function, args, startTime, "", "PutObject failed", err)
  2915. return
  2916. }
  2917. // read the data back
  2918. r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  2919. if err != nil {
  2920. logError(testName, function, args, startTime, "", "PutObject failed", err)
  2921. return
  2922. }
  2923. offset := int64(2048)
  2924. // read directly
  2925. buf1 := make([]byte, 512)
  2926. buf2 := make([]byte, 512)
  2927. buf3 := make([]byte, 512)
  2928. buf4 := make([]byte, 512)
  2929. // Test readAt before stat is called such that objectInfo doesn't change.
  2930. m, err := r.ReadAt(buf1, offset)
  2931. if err != nil {
  2932. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  2933. return
  2934. }
  2935. if m != len(buf1) {
  2936. logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err)
  2937. return
  2938. }
  2939. if !bytes.Equal(buf1, buf[offset:offset+512]) {
  2940. logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
  2941. return
  2942. }
  2943. offset += 512
  2944. st, err := r.Stat()
  2945. if err != nil {
  2946. logError(testName, function, args, startTime, "", "Stat failed", err)
  2947. return
  2948. }
  2949. if st.Size != int64(bufSize) {
  2950. logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err)
  2951. return
  2952. }
  2953. m, err = r.ReadAt(buf2, offset)
  2954. if err != nil {
  2955. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  2956. return
  2957. }
  2958. if m != len(buf2) {
  2959. logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err)
  2960. return
  2961. }
  2962. if !bytes.Equal(buf2, buf[offset:offset+512]) {
  2963. logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
  2964. return
  2965. }
  2966. offset += 512
  2967. m, err = r.ReadAt(buf3, offset)
  2968. if err != nil {
  2969. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  2970. return
  2971. }
  2972. if m != len(buf3) {
  2973. logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err)
  2974. return
  2975. }
  2976. if !bytes.Equal(buf3, buf[offset:offset+512]) {
  2977. logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
  2978. return
  2979. }
  2980. offset += 512
  2981. m, err = r.ReadAt(buf4, offset)
  2982. if err != nil {
  2983. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  2984. return
  2985. }
  2986. if m != len(buf4) {
  2987. logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err)
  2988. return
  2989. }
  2990. if !bytes.Equal(buf4, buf[offset:offset+512]) {
  2991. logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
  2992. return
  2993. }
  2994. buf5 := make([]byte, len(buf))
  2995. // Read the whole object.
  2996. m, err = r.ReadAt(buf5, 0)
  2997. if err != nil {
  2998. if err != io.EOF {
  2999. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  3000. return
  3001. }
  3002. }
  3003. if m != len(buf5) {
  3004. logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err)
  3005. return
  3006. }
  3007. if !bytes.Equal(buf, buf5) {
  3008. logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err)
  3009. return
  3010. }
  3011. buf6 := make([]byte, len(buf)+1)
  3012. // Read the whole object and beyond.
  3013. _, err = r.ReadAt(buf6, 0)
  3014. if err != nil {
  3015. if err != io.EOF {
  3016. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  3017. return
  3018. }
  3019. }
  3020. successLogger(testName, function, args, startTime).Info()
  3021. }
  3022. // Reproduces issue https://github.com/minio/minio-go/issues/1137
  3023. func testGetObjectReadAtWhenEOFWasReached() {
  3024. // initialize logging params
  3025. startTime := time.Now()
  3026. testName := getFuncName()
  3027. function := "GetObject(bucketName, objectName)"
  3028. args := map[string]interface{}{}
  3029. // Seed random based on current time.
  3030. rand.Seed(time.Now().Unix())
  3031. // Instantiate new minio client object.
  3032. c, err := minio.New(os.Getenv(serverEndpoint),
  3033. &minio.Options{
  3034. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  3035. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  3036. })
  3037. if err != nil {
  3038. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  3039. return
  3040. }
  3041. // Enable tracing, write to stderr.
  3042. // c.TraceOn(os.Stderr)
  3043. // Set user agent.
  3044. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  3045. // Generate a new random bucket name.
  3046. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  3047. args["bucketName"] = bucketName
  3048. // Make a new bucket.
  3049. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  3050. if err != nil {
  3051. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  3052. return
  3053. }
  3054. defer cleanupBucket(bucketName, c)
  3055. // Generate 33K of data.
  3056. bufSize := dataFileMap["datafile-33-kB"]
  3057. var reader = getDataReader("datafile-33-kB")
  3058. defer reader.Close()
  3059. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  3060. args["objectName"] = objectName
  3061. buf, err := ioutil.ReadAll(reader)
  3062. if err != nil {
  3063. logError(testName, function, args, startTime, "", "ReadAll failed", err)
  3064. return
  3065. }
  3066. // Save the data
  3067. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  3068. if err != nil {
  3069. logError(testName, function, args, startTime, "", "PutObject failed", err)
  3070. return
  3071. }
  3072. // read the data back
  3073. r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  3074. if err != nil {
  3075. logError(testName, function, args, startTime, "", "PutObject failed", err)
  3076. return
  3077. }
  3078. // read directly
  3079. buf1 := make([]byte, len(buf))
  3080. buf2 := make([]byte, 512)
  3081. m, err := r.Read(buf1)
  3082. if err != nil {
  3083. if err != io.EOF {
  3084. logError(testName, function, args, startTime, "", "Read failed", err)
  3085. return
  3086. }
  3087. }
  3088. if m != len(buf1) {
  3089. logError(testName, function, args, startTime, "", "Read read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err)
  3090. return
  3091. }
  3092. if !bytes.Equal(buf1, buf) {
  3093. logError(testName, function, args, startTime, "", "Incorrect count of Read data", err)
  3094. return
  3095. }
  3096. st, err := r.Stat()
  3097. if err != nil {
  3098. logError(testName, function, args, startTime, "", "Stat failed", err)
  3099. return
  3100. }
  3101. if st.Size != int64(bufSize) {
  3102. logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err)
  3103. return
  3104. }
  3105. m, err = r.ReadAt(buf2, 512)
  3106. if err != nil {
  3107. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  3108. return
  3109. }
  3110. if m != len(buf2) {
  3111. logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err)
  3112. return
  3113. }
  3114. if !bytes.Equal(buf2, buf[512:1024]) {
  3115. logError(testName, function, args, startTime, "", "Incorrect count of ReadAt data", err)
  3116. return
  3117. }
  3118. successLogger(testName, function, args, startTime).Info()
  3119. }
  3120. // Test Presigned Post Policy
  3121. func testPresignedPostPolicy() {
  3122. // initialize logging params
  3123. startTime := time.Now()
  3124. testName := getFuncName()
  3125. function := "PresignedPostPolicy(policy)"
  3126. args := map[string]interface{}{
  3127. "policy": "",
  3128. }
  3129. // Seed random based on current time.
  3130. rand.Seed(time.Now().Unix())
  3131. // Instantiate new minio client object
  3132. c, err := minio.New(os.Getenv(serverEndpoint),
  3133. &minio.Options{
  3134. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  3135. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  3136. })
  3137. if err != nil {
  3138. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  3139. return
  3140. }
  3141. // Enable tracing, write to stderr.
  3142. // c.TraceOn(os.Stderr)
  3143. // Set user agent.
  3144. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  3145. // Generate a new random bucket name.
  3146. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  3147. // Make a new bucket in 'us-east-1' (source bucket).
  3148. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  3149. if err != nil {
  3150. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  3151. return
  3152. }
  3153. defer cleanupBucket(bucketName, c)
  3154. // Generate 33K of data.
  3155. var reader = getDataReader("datafile-33-kB")
  3156. defer reader.Close()
  3157. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  3158. // Azure requires the key to not start with a number
  3159. metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "user")
  3160. metadataValue := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  3161. buf, err := ioutil.ReadAll(reader)
  3162. if err != nil {
  3163. logError(testName, function, args, startTime, "", "ReadAll failed", err)
  3164. return
  3165. }
  3166. // Save the data
  3167. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  3168. if err != nil {
  3169. logError(testName, function, args, startTime, "", "PutObject failed", err)
  3170. return
  3171. }
  3172. policy := minio.NewPostPolicy()
  3173. if err := policy.SetBucket(""); err == nil {
  3174. logError(testName, function, args, startTime, "", "SetBucket did not fail for invalid conditions", err)
  3175. return
  3176. }
  3177. if err := policy.SetKey(""); err == nil {
  3178. logError(testName, function, args, startTime, "", "SetKey did not fail for invalid conditions", err)
  3179. return
  3180. }
  3181. if err := policy.SetKeyStartsWith(""); err == nil {
  3182. logError(testName, function, args, startTime, "", "SetKeyStartsWith did not fail for invalid conditions", err)
  3183. return
  3184. }
  3185. if err := policy.SetExpires(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)); err == nil {
  3186. logError(testName, function, args, startTime, "", "SetExpires did not fail for invalid conditions", err)
  3187. return
  3188. }
  3189. if err := policy.SetContentType(""); err == nil {
  3190. logError(testName, function, args, startTime, "", "SetContentType did not fail for invalid conditions", err)
  3191. return
  3192. }
  3193. if err := policy.SetContentTypeStartsWith(""); err == nil {
  3194. logError(testName, function, args, startTime, "", "SetContentTypeStartsWith did not fail for invalid conditions", err)
  3195. return
  3196. }
  3197. if err := policy.SetContentLengthRange(1024*1024, 1024); err == nil {
  3198. logError(testName, function, args, startTime, "", "SetContentLengthRange did not fail for invalid conditions", err)
  3199. return
  3200. }
  3201. if err := policy.SetUserMetadata("", ""); err == nil {
  3202. logError(testName, function, args, startTime, "", "SetUserMetadata did not fail for invalid conditions", err)
  3203. return
  3204. }
  3205. policy.SetBucket(bucketName)
  3206. policy.SetKey(objectName)
  3207. policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days
  3208. policy.SetContentType("binary/octet-stream")
  3209. policy.SetContentLengthRange(10, 1024*1024)
  3210. policy.SetUserMetadata(metadataKey, metadataValue)
  3211. args["policy"] = policy.String()
  3212. presignedPostPolicyURL, formData, err := c.PresignedPostPolicy(context.Background(), policy)
  3213. if err != nil {
  3214. logError(testName, function, args, startTime, "", "PresignedPostPolicy failed", err)
  3215. return
  3216. }
  3217. var formBuf bytes.Buffer
  3218. writer := multipart.NewWriter(&formBuf)
  3219. for k, v := range formData {
  3220. writer.WriteField(k, v)
  3221. }
  3222. // Get a 33KB file to upload and test if set post policy works
  3223. var filePath = getMintDataDirFilePath("datafile-33-kB")
  3224. if filePath == "" {
  3225. // Make a temp file with 33 KB data.
  3226. file, err := ioutil.TempFile(os.TempDir(), "PresignedPostPolicyTest")
  3227. if err != nil {
  3228. logError(testName, function, args, startTime, "", "TempFile creation failed", err)
  3229. return
  3230. }
  3231. if _, err = io.Copy(file, getDataReader("datafile-33-kB")); err != nil {
  3232. logError(testName, function, args, startTime, "", "Copy failed", err)
  3233. return
  3234. }
  3235. if err = file.Close(); err != nil {
  3236. logError(testName, function, args, startTime, "", "File Close failed", err)
  3237. return
  3238. }
  3239. filePath = file.Name()
  3240. }
  3241. // add file to post request
  3242. f, err := os.Open(filePath)
  3243. defer f.Close()
  3244. if err != nil {
  3245. logError(testName, function, args, startTime, "", "File open failed", err)
  3246. return
  3247. }
  3248. w, err := writer.CreateFormFile("file", filePath)
  3249. if err != nil {
  3250. logError(testName, function, args, startTime, "", "CreateFormFile failed", err)
  3251. return
  3252. }
  3253. _, err = io.Copy(w, f)
  3254. if err != nil {
  3255. logError(testName, function, args, startTime, "", "Copy failed", err)
  3256. return
  3257. }
  3258. writer.Close()
  3259. transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS)))
  3260. if err != nil {
  3261. logError(testName, function, args, startTime, "", "DefaultTransport failed", err)
  3262. return
  3263. }
  3264. httpClient := &http.Client{
  3265. // Setting a sensible time out of 30secs to wait for response
  3266. // headers. Request is pro-actively canceled after 30secs
  3267. // with no response.
  3268. Timeout: 30 * time.Second,
  3269. Transport: transport,
  3270. }
  3271. req, err := http.NewRequest(http.MethodPost, presignedPostPolicyURL.String(), bytes.NewReader(formBuf.Bytes()))
  3272. if err != nil {
  3273. logError(testName, function, args, startTime, "", "Http request failed", err)
  3274. return
  3275. }
  3276. req.Header.Set("Content-Type", writer.FormDataContentType())
  3277. // make post request with correct form data
  3278. res, err := httpClient.Do(req)
  3279. if err != nil {
  3280. logError(testName, function, args, startTime, "", "Http request failed", err)
  3281. return
  3282. }
  3283. defer res.Body.Close()
  3284. if res.StatusCode != http.StatusNoContent {
  3285. logError(testName, function, args, startTime, "", "Http request failed", errors.New(res.Status))
  3286. return
  3287. }
  3288. // expected path should be absolute path of the object
  3289. var scheme string
  3290. if mustParseBool(os.Getenv(enableHTTPS)) {
  3291. scheme = "https://"
  3292. } else {
  3293. scheme = "http://"
  3294. }
  3295. expectedLocation := scheme + os.Getenv(serverEndpoint) + "/" + bucketName + "/" + objectName
  3296. expectedLocationBucketDNS := scheme + bucketName + "." + os.Getenv(serverEndpoint) + "/" + objectName
  3297. if val, ok := res.Header["Location"]; ok {
  3298. if val[0] != expectedLocation && val[0] != expectedLocationBucketDNS {
  3299. logError(testName, function, args, startTime, "", "Location in header response is incorrect", err)
  3300. return
  3301. }
  3302. } else {
  3303. logError(testName, function, args, startTime, "", "Location not found in header response", err)
  3304. return
  3305. }
  3306. successLogger(testName, function, args, startTime).Info()
  3307. }
  3308. // Tests copy object
  3309. func testCopyObject() {
  3310. // initialize logging params
  3311. startTime := time.Now()
  3312. testName := getFuncName()
  3313. function := "CopyObject(dst, src)"
  3314. args := map[string]interface{}{}
  3315. // Seed random based on current time.
  3316. rand.Seed(time.Now().Unix())
  3317. // Instantiate new minio client object
  3318. c, err := minio.New(os.Getenv(serverEndpoint),
  3319. &minio.Options{
  3320. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  3321. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  3322. })
  3323. if err != nil {
  3324. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  3325. return
  3326. }
  3327. // Enable tracing, write to stderr.
  3328. // c.TraceOn(os.Stderr)
  3329. // Set user agent.
  3330. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  3331. // Generate a new random bucket name.
  3332. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  3333. // Make a new bucket in 'us-east-1' (source bucket).
  3334. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  3335. if err != nil {
  3336. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  3337. return
  3338. }
  3339. defer cleanupBucket(bucketName, c)
  3340. // Make a new bucket in 'us-east-1' (destination bucket).
  3341. err = c.MakeBucket(context.Background(), bucketName+"-copy", minio.MakeBucketOptions{Region: "us-east-1"})
  3342. if err != nil {
  3343. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  3344. return
  3345. }
  3346. defer cleanupBucket(bucketName+"-copy", c)
  3347. // Generate 33K of data.
  3348. bufSize := dataFileMap["datafile-33-kB"]
  3349. var reader = getDataReader("datafile-33-kB")
  3350. // Save the data
  3351. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  3352. _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  3353. if err != nil {
  3354. logError(testName, function, args, startTime, "", "PutObject failed", err)
  3355. return
  3356. }
  3357. r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  3358. if err != nil {
  3359. logError(testName, function, args, startTime, "", "GetObject failed", err)
  3360. return
  3361. }
  3362. // Check the various fields of source object against destination object.
  3363. objInfo, err := r.Stat()
  3364. if err != nil {
  3365. logError(testName, function, args, startTime, "", "Stat failed", err)
  3366. return
  3367. }
  3368. // Copy Source
  3369. src := minio.CopySrcOptions{
  3370. Bucket: bucketName,
  3371. Object: objectName,
  3372. // Set copy conditions.
  3373. MatchETag: objInfo.ETag,
  3374. MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC),
  3375. }
  3376. args["src"] = src
  3377. dst := minio.CopyDestOptions{
  3378. Bucket: bucketName + "-copy",
  3379. Object: objectName + "-copy",
  3380. }
  3381. // Perform the Copy
  3382. if _, err = c.CopyObject(context.Background(), dst, src); err != nil {
  3383. logError(testName, function, args, startTime, "", "CopyObject failed", err)
  3384. return
  3385. }
  3386. // Source object
  3387. r, err = c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  3388. if err != nil {
  3389. logError(testName, function, args, startTime, "", "GetObject failed", err)
  3390. return
  3391. }
  3392. // Destination object
  3393. readerCopy, err := c.GetObject(context.Background(), bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{})
  3394. if err != nil {
  3395. logError(testName, function, args, startTime, "", "GetObject failed", err)
  3396. return
  3397. }
  3398. // Check the various fields of source object against destination object.
  3399. objInfo, err = r.Stat()
  3400. if err != nil {
  3401. logError(testName, function, args, startTime, "", "Stat failed", err)
  3402. return
  3403. }
  3404. objInfoCopy, err := readerCopy.Stat()
  3405. if err != nil {
  3406. logError(testName, function, args, startTime, "", "Stat failed", err)
  3407. return
  3408. }
  3409. if objInfo.Size != objInfoCopy.Size {
  3410. logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(objInfoCopy.Size)+", got "+string(objInfo.Size), err)
  3411. return
  3412. }
  3413. if err := crcMatchesName(r, "datafile-33-kB"); err != nil {
  3414. logError(testName, function, args, startTime, "", "data CRC check failed", err)
  3415. return
  3416. }
  3417. if err := crcMatchesName(readerCopy, "datafile-33-kB"); err != nil {
  3418. logError(testName, function, args, startTime, "", "copy data CRC check failed", err)
  3419. return
  3420. }
  3421. // Close all the get readers before proceeding with CopyObject operations.
  3422. r.Close()
  3423. readerCopy.Close()
  3424. // CopyObject again but with wrong conditions
  3425. src = minio.CopySrcOptions{
  3426. Bucket: bucketName,
  3427. Object: objectName,
  3428. MatchUnmodifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC),
  3429. NoMatchETag: objInfo.ETag,
  3430. }
  3431. // Perform the Copy which should fail
  3432. _, err = c.CopyObject(context.Background(), dst, src)
  3433. if err == nil {
  3434. logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err)
  3435. return
  3436. }
  3437. src = minio.CopySrcOptions{
  3438. Bucket: bucketName,
  3439. Object: objectName,
  3440. }
  3441. dst = minio.CopyDestOptions{
  3442. Bucket: bucketName,
  3443. Object: objectName,
  3444. ReplaceMetadata: true,
  3445. UserMetadata: map[string]string{
  3446. "Copy": "should be same",
  3447. },
  3448. }
  3449. args["dst"] = dst
  3450. args["src"] = src
  3451. _, err = c.CopyObject(context.Background(), dst, src)
  3452. if err != nil {
  3453. logError(testName, function, args, startTime, "", "CopyObject shouldn't fail", err)
  3454. return
  3455. }
  3456. oi, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
  3457. if err != nil {
  3458. logError(testName, function, args, startTime, "", "StatObject failed", err)
  3459. return
  3460. }
  3461. stOpts := minio.StatObjectOptions{}
  3462. stOpts.SetMatchETag(oi.ETag)
  3463. objInfo, err = c.StatObject(context.Background(), bucketName, objectName, stOpts)
  3464. if err != nil {
  3465. logError(testName, function, args, startTime, "", "CopyObject ETag should match and not fail", err)
  3466. return
  3467. }
  3468. if objInfo.Metadata.Get("x-amz-meta-copy") != "should be same" {
  3469. logError(testName, function, args, startTime, "", "CopyObject modified metadata should match", err)
  3470. return
  3471. }
  3472. successLogger(testName, function, args, startTime).Info()
  3473. }
  3474. // Tests SSE-C get object ReaderSeeker interface methods.
  3475. func testSSECEncryptedGetObjectReadSeekFunctional() {
  3476. // initialize logging params
  3477. startTime := time.Now()
  3478. testName := getFuncName()
  3479. function := "GetObject(bucketName, objectName)"
  3480. args := map[string]interface{}{}
  3481. // Seed random based on current time.
  3482. rand.Seed(time.Now().Unix())
  3483. // Instantiate new minio client object.
  3484. c, err := minio.New(os.Getenv(serverEndpoint),
  3485. &minio.Options{
  3486. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  3487. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  3488. })
  3489. if err != nil {
  3490. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  3491. return
  3492. }
  3493. // Enable tracing, write to stderr.
  3494. // c.TraceOn(os.Stderr)
  3495. // Set user agent.
  3496. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  3497. // Generate a new random bucket name.
  3498. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  3499. args["bucketName"] = bucketName
  3500. // Make a new bucket.
  3501. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  3502. if err != nil {
  3503. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  3504. return
  3505. }
  3506. defer func() {
  3507. // Delete all objects and buckets
  3508. if err = cleanupBucket(bucketName, c); err != nil {
  3509. logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
  3510. return
  3511. }
  3512. }()
  3513. // Generate 129MiB of data.
  3514. bufSize := dataFileMap["datafile-129-MB"]
  3515. var reader = getDataReader("datafile-129-MB")
  3516. defer reader.Close()
  3517. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  3518. args["objectName"] = objectName
  3519. buf, err := ioutil.ReadAll(reader)
  3520. if err != nil {
  3521. logError(testName, function, args, startTime, "", "ReadAll failed", err)
  3522. return
  3523. }
  3524. // Save the data
  3525. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{
  3526. ContentType: "binary/octet-stream",
  3527. ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)),
  3528. })
  3529. if err != nil {
  3530. logError(testName, function, args, startTime, "", "PutObject failed", err)
  3531. return
  3532. }
  3533. // Read the data back
  3534. r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{
  3535. ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)),
  3536. })
  3537. if err != nil {
  3538. logError(testName, function, args, startTime, "", "GetObject failed", err)
  3539. return
  3540. }
  3541. defer r.Close()
  3542. st, err := r.Stat()
  3543. if err != nil {
  3544. logError(testName, function, args, startTime, "", "Stat object failed", err)
  3545. return
  3546. }
  3547. if st.Size != int64(bufSize) {
  3548. logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err)
  3549. return
  3550. }
  3551. // This following function helps us to compare data from the reader after seek
  3552. // with the data from the original buffer
  3553. cmpData := func(r io.Reader, start, end int) {
  3554. if end-start == 0 {
  3555. return
  3556. }
  3557. buffer := bytes.NewBuffer([]byte{})
  3558. if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil {
  3559. if err != io.EOF {
  3560. logError(testName, function, args, startTime, "", "CopyN failed", err)
  3561. return
  3562. }
  3563. }
  3564. if !bytes.Equal(buf[start:end], buffer.Bytes()) {
  3565. logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err)
  3566. return
  3567. }
  3568. }
  3569. testCases := []struct {
  3570. offset int64
  3571. whence int
  3572. pos int64
  3573. err error
  3574. shouldCmp bool
  3575. start int
  3576. end int
  3577. }{
  3578. // Start from offset 0, fetch data and compare
  3579. {0, 0, 0, nil, true, 0, 0},
  3580. // Start from offset 2048, fetch data and compare
  3581. {2048, 0, 2048, nil, true, 2048, bufSize},
  3582. // Start from offset larger than possible
  3583. {int64(bufSize) + 1024, 0, 0, io.EOF, false, 0, 0},
  3584. // Move to offset 0 without comparing
  3585. {0, 0, 0, nil, false, 0, 0},
  3586. // Move one step forward and compare
  3587. {1, 1, 1, nil, true, 1, bufSize},
  3588. // Move larger than possible
  3589. {int64(bufSize), 1, 0, io.EOF, false, 0, 0},
  3590. // Provide negative offset with CUR_SEEK
  3591. {int64(-1), 1, 0, fmt.Errorf("Negative position not allowed for 1"), false, 0, 0},
  3592. // Test with whence SEEK_END and with positive offset
  3593. {1024, 2, 0, io.EOF, false, 0, 0},
  3594. // Test with whence SEEK_END and with negative offset
  3595. {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize},
  3596. // Test with whence SEEK_END and with large negative offset
  3597. {-int64(bufSize) * 2, 2, 0, fmt.Errorf("Seeking at negative offset not allowed for 2"), false, 0, 0},
  3598. // Test with invalid whence
  3599. {0, 3, 0, fmt.Errorf("Invalid whence 3"), false, 0, 0},
  3600. }
  3601. for i, testCase := range testCases {
  3602. // Perform seek operation
  3603. n, err := r.Seek(testCase.offset, testCase.whence)
  3604. if err != nil && testCase.err == nil {
  3605. // We expected success.
  3606. logError(testName, function, args, startTime, "",
  3607. fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err)
  3608. return
  3609. }
  3610. if err == nil && testCase.err != nil {
  3611. // We expected failure, but got success.
  3612. logError(testName, function, args, startTime, "",
  3613. fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err)
  3614. return
  3615. }
  3616. if err != nil && testCase.err != nil {
  3617. if err.Error() != testCase.err.Error() {
  3618. // We expect a specific error
  3619. logError(testName, function, args, startTime, "",
  3620. fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err)
  3621. return
  3622. }
  3623. }
  3624. // Check the returned seek pos
  3625. if n != testCase.pos {
  3626. logError(testName, function, args, startTime, "",
  3627. fmt.Sprintf("Test %d, number of bytes seeked does not match, expected %d, got %d", i+1, testCase.pos, n), err)
  3628. return
  3629. }
  3630. // Compare only if shouldCmp is activated
  3631. if testCase.shouldCmp {
  3632. cmpData(r, testCase.start, testCase.end)
  3633. }
  3634. }
  3635. successLogger(testName, function, args, startTime).Info()
  3636. }
  3637. // Tests SSE-S3 get object ReaderSeeker interface methods.
  3638. func testSSES3EncryptedGetObjectReadSeekFunctional() {
  3639. // initialize logging params
  3640. startTime := time.Now()
  3641. testName := getFuncName()
  3642. function := "GetObject(bucketName, objectName)"
  3643. args := map[string]interface{}{}
  3644. // Seed random based on current time.
  3645. rand.Seed(time.Now().Unix())
  3646. // Instantiate new minio client object.
  3647. c, err := minio.New(os.Getenv(serverEndpoint),
  3648. &minio.Options{
  3649. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  3650. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  3651. })
  3652. if err != nil {
  3653. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  3654. return
  3655. }
  3656. // Enable tracing, write to stderr.
  3657. // c.TraceOn(os.Stderr)
  3658. // Set user agent.
  3659. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  3660. // Generate a new random bucket name.
  3661. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  3662. args["bucketName"] = bucketName
  3663. // Make a new bucket.
  3664. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  3665. if err != nil {
  3666. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  3667. return
  3668. }
  3669. defer func() {
  3670. // Delete all objects and buckets
  3671. if err = cleanupBucket(bucketName, c); err != nil {
  3672. logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
  3673. return
  3674. }
  3675. }()
  3676. // Generate 129MiB of data.
  3677. bufSize := dataFileMap["datafile-129-MB"]
  3678. var reader = getDataReader("datafile-129-MB")
  3679. defer reader.Close()
  3680. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  3681. args["objectName"] = objectName
  3682. buf, err := ioutil.ReadAll(reader)
  3683. if err != nil {
  3684. logError(testName, function, args, startTime, "", "ReadAll failed", err)
  3685. return
  3686. }
  3687. // Save the data
  3688. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{
  3689. ContentType: "binary/octet-stream",
  3690. ServerSideEncryption: encrypt.NewSSE(),
  3691. })
  3692. if err != nil {
  3693. logError(testName, function, args, startTime, "", "PutObject failed", err)
  3694. return
  3695. }
  3696. // Read the data back
  3697. r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  3698. if err != nil {
  3699. logError(testName, function, args, startTime, "", "GetObject failed", err)
  3700. return
  3701. }
  3702. defer r.Close()
  3703. st, err := r.Stat()
  3704. if err != nil {
  3705. logError(testName, function, args, startTime, "", "Stat object failed", err)
  3706. return
  3707. }
  3708. if st.Size != int64(bufSize) {
  3709. logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err)
  3710. return
  3711. }
  3712. // This following function helps us to compare data from the reader after seek
  3713. // with the data from the original buffer
  3714. cmpData := func(r io.Reader, start, end int) {
  3715. if end-start == 0 {
  3716. return
  3717. }
  3718. buffer := bytes.NewBuffer([]byte{})
  3719. if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil {
  3720. if err != io.EOF {
  3721. logError(testName, function, args, startTime, "", "CopyN failed", err)
  3722. return
  3723. }
  3724. }
  3725. if !bytes.Equal(buf[start:end], buffer.Bytes()) {
  3726. logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err)
  3727. return
  3728. }
  3729. }
  3730. testCases := []struct {
  3731. offset int64
  3732. whence int
  3733. pos int64
  3734. err error
  3735. shouldCmp bool
  3736. start int
  3737. end int
  3738. }{
  3739. // Start from offset 0, fetch data and compare
  3740. {0, 0, 0, nil, true, 0, 0},
  3741. // Start from offset 2048, fetch data and compare
  3742. {2048, 0, 2048, nil, true, 2048, bufSize},
  3743. // Start from offset larger than possible
  3744. {int64(bufSize) + 1024, 0, 0, io.EOF, false, 0, 0},
  3745. // Move to offset 0 without comparing
  3746. {0, 0, 0, nil, false, 0, 0},
  3747. // Move one step forward and compare
  3748. {1, 1, 1, nil, true, 1, bufSize},
  3749. // Move larger than possible
  3750. {int64(bufSize), 1, 0, io.EOF, false, 0, 0},
  3751. // Provide negative offset with CUR_SEEK
  3752. {int64(-1), 1, 0, fmt.Errorf("Negative position not allowed for 1"), false, 0, 0},
  3753. // Test with whence SEEK_END and with positive offset
  3754. {1024, 2, 0, io.EOF, false, 0, 0},
  3755. // Test with whence SEEK_END and with negative offset
  3756. {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize},
  3757. // Test with whence SEEK_END and with large negative offset
  3758. {-int64(bufSize) * 2, 2, 0, fmt.Errorf("Seeking at negative offset not allowed for 2"), false, 0, 0},
  3759. // Test with invalid whence
  3760. {0, 3, 0, fmt.Errorf("Invalid whence 3"), false, 0, 0},
  3761. }
  3762. for i, testCase := range testCases {
  3763. // Perform seek operation
  3764. n, err := r.Seek(testCase.offset, testCase.whence)
  3765. if err != nil && testCase.err == nil {
  3766. // We expected success.
  3767. logError(testName, function, args, startTime, "",
  3768. fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err)
  3769. return
  3770. }
  3771. if err == nil && testCase.err != nil {
  3772. // We expected failure, but got success.
  3773. logError(testName, function, args, startTime, "",
  3774. fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err)
  3775. return
  3776. }
  3777. if err != nil && testCase.err != nil {
  3778. if err.Error() != testCase.err.Error() {
  3779. // We expect a specific error
  3780. logError(testName, function, args, startTime, "",
  3781. fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err)
  3782. return
  3783. }
  3784. }
  3785. // Check the returned seek pos
  3786. if n != testCase.pos {
  3787. logError(testName, function, args, startTime, "",
  3788. fmt.Sprintf("Test %d, number of bytes seeked does not match, expected %d, got %d", i+1, testCase.pos, n), err)
  3789. return
  3790. }
  3791. // Compare only if shouldCmp is activated
  3792. if testCase.shouldCmp {
  3793. cmpData(r, testCase.start, testCase.end)
  3794. }
  3795. }
  3796. successLogger(testName, function, args, startTime).Info()
  3797. }
  3798. // Tests SSE-C get object ReaderAt interface methods.
  3799. func testSSECEncryptedGetObjectReadAtFunctional() {
  3800. // initialize logging params
  3801. startTime := time.Now()
  3802. testName := getFuncName()
  3803. function := "GetObject(bucketName, objectName)"
  3804. args := map[string]interface{}{}
  3805. // Seed random based on current time.
  3806. rand.Seed(time.Now().Unix())
  3807. // Instantiate new minio client object.
  3808. c, err := minio.New(os.Getenv(serverEndpoint),
  3809. &minio.Options{
  3810. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  3811. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  3812. })
  3813. if err != nil {
  3814. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  3815. return
  3816. }
  3817. // Enable tracing, write to stderr.
  3818. // c.TraceOn(os.Stderr)
  3819. // Set user agent.
  3820. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  3821. // Generate a new random bucket name.
  3822. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  3823. args["bucketName"] = bucketName
  3824. // Make a new bucket.
  3825. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  3826. if err != nil {
  3827. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  3828. return
  3829. }
  3830. defer cleanupBucket(bucketName, c)
  3831. // Generate 129MiB of data.
  3832. bufSize := dataFileMap["datafile-129-MB"]
  3833. var reader = getDataReader("datafile-129-MB")
  3834. defer reader.Close()
  3835. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  3836. args["objectName"] = objectName
  3837. buf, err := ioutil.ReadAll(reader)
  3838. if err != nil {
  3839. logError(testName, function, args, startTime, "", "ReadAll failed", err)
  3840. return
  3841. }
  3842. // Save the data
  3843. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{
  3844. ContentType: "binary/octet-stream",
  3845. ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)),
  3846. })
  3847. if err != nil {
  3848. logError(testName, function, args, startTime, "", "PutObject failed", err)
  3849. return
  3850. }
  3851. // read the data back
  3852. r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{
  3853. ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)),
  3854. })
  3855. if err != nil {
  3856. logError(testName, function, args, startTime, "", "PutObject failed", err)
  3857. return
  3858. }
  3859. defer r.Close()
  3860. offset := int64(2048)
  3861. // read directly
  3862. buf1 := make([]byte, 512)
  3863. buf2 := make([]byte, 512)
  3864. buf3 := make([]byte, 512)
  3865. buf4 := make([]byte, 512)
  3866. // Test readAt before stat is called such that objectInfo doesn't change.
  3867. m, err := r.ReadAt(buf1, offset)
  3868. if err != nil {
  3869. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  3870. return
  3871. }
  3872. if m != len(buf1) {
  3873. logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err)
  3874. return
  3875. }
  3876. if !bytes.Equal(buf1, buf[offset:offset+512]) {
  3877. logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
  3878. return
  3879. }
  3880. offset += 512
  3881. st, err := r.Stat()
  3882. if err != nil {
  3883. logError(testName, function, args, startTime, "", "Stat failed", err)
  3884. return
  3885. }
  3886. if st.Size != int64(bufSize) {
  3887. logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err)
  3888. return
  3889. }
  3890. m, err = r.ReadAt(buf2, offset)
  3891. if err != nil {
  3892. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  3893. return
  3894. }
  3895. if m != len(buf2) {
  3896. logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err)
  3897. return
  3898. }
  3899. if !bytes.Equal(buf2, buf[offset:offset+512]) {
  3900. logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
  3901. return
  3902. }
  3903. offset += 512
  3904. m, err = r.ReadAt(buf3, offset)
  3905. if err != nil {
  3906. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  3907. return
  3908. }
  3909. if m != len(buf3) {
  3910. logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err)
  3911. return
  3912. }
  3913. if !bytes.Equal(buf3, buf[offset:offset+512]) {
  3914. logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
  3915. return
  3916. }
  3917. offset += 512
  3918. m, err = r.ReadAt(buf4, offset)
  3919. if err != nil {
  3920. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  3921. return
  3922. }
  3923. if m != len(buf4) {
  3924. logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err)
  3925. return
  3926. }
  3927. if !bytes.Equal(buf4, buf[offset:offset+512]) {
  3928. logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
  3929. return
  3930. }
  3931. buf5 := make([]byte, len(buf))
  3932. // Read the whole object.
  3933. m, err = r.ReadAt(buf5, 0)
  3934. if err != nil {
  3935. if err != io.EOF {
  3936. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  3937. return
  3938. }
  3939. }
  3940. if m != len(buf5) {
  3941. logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err)
  3942. return
  3943. }
  3944. if !bytes.Equal(buf, buf5) {
  3945. logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err)
  3946. return
  3947. }
  3948. buf6 := make([]byte, len(buf)+1)
  3949. // Read the whole object and beyond.
  3950. _, err = r.ReadAt(buf6, 0)
  3951. if err != nil {
  3952. if err != io.EOF {
  3953. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  3954. return
  3955. }
  3956. }
  3957. successLogger(testName, function, args, startTime).Info()
  3958. }
  3959. // Tests SSE-S3 get object ReaderAt interface methods.
  3960. func testSSES3EncryptedGetObjectReadAtFunctional() {
  3961. // initialize logging params
  3962. startTime := time.Now()
  3963. testName := getFuncName()
  3964. function := "GetObject(bucketName, objectName)"
  3965. args := map[string]interface{}{}
  3966. // Seed random based on current time.
  3967. rand.Seed(time.Now().Unix())
  3968. // Instantiate new minio client object.
  3969. c, err := minio.New(os.Getenv(serverEndpoint),
  3970. &minio.Options{
  3971. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  3972. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  3973. })
  3974. if err != nil {
  3975. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  3976. return
  3977. }
  3978. // Enable tracing, write to stderr.
  3979. // c.TraceOn(os.Stderr)
  3980. // Set user agent.
  3981. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  3982. // Generate a new random bucket name.
  3983. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  3984. args["bucketName"] = bucketName
  3985. // Make a new bucket.
  3986. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  3987. if err != nil {
  3988. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  3989. return
  3990. }
  3991. defer cleanupBucket(bucketName, c)
  3992. // Generate 129MiB of data.
  3993. bufSize := dataFileMap["datafile-129-MB"]
  3994. var reader = getDataReader("datafile-129-MB")
  3995. defer reader.Close()
  3996. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  3997. args["objectName"] = objectName
  3998. buf, err := ioutil.ReadAll(reader)
  3999. if err != nil {
  4000. logError(testName, function, args, startTime, "", "ReadAll failed", err)
  4001. return
  4002. }
  4003. // Save the data
  4004. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{
  4005. ContentType: "binary/octet-stream",
  4006. ServerSideEncryption: encrypt.NewSSE(),
  4007. })
  4008. if err != nil {
  4009. logError(testName, function, args, startTime, "", "PutObject failed", err)
  4010. return
  4011. }
  4012. // read the data back
  4013. r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  4014. if err != nil {
  4015. logError(testName, function, args, startTime, "", "PutObject failed", err)
  4016. return
  4017. }
  4018. defer r.Close()
  4019. offset := int64(2048)
  4020. // read directly
  4021. buf1 := make([]byte, 512)
  4022. buf2 := make([]byte, 512)
  4023. buf3 := make([]byte, 512)
  4024. buf4 := make([]byte, 512)
  4025. // Test readAt before stat is called such that objectInfo doesn't change.
  4026. m, err := r.ReadAt(buf1, offset)
  4027. if err != nil {
  4028. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  4029. return
  4030. }
  4031. if m != len(buf1) {
  4032. logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err)
  4033. return
  4034. }
  4035. if !bytes.Equal(buf1, buf[offset:offset+512]) {
  4036. logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
  4037. return
  4038. }
  4039. offset += 512
  4040. st, err := r.Stat()
  4041. if err != nil {
  4042. logError(testName, function, args, startTime, "", "Stat failed", err)
  4043. return
  4044. }
  4045. if st.Size != int64(bufSize) {
  4046. logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err)
  4047. return
  4048. }
  4049. m, err = r.ReadAt(buf2, offset)
  4050. if err != nil {
  4051. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  4052. return
  4053. }
  4054. if m != len(buf2) {
  4055. logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err)
  4056. return
  4057. }
  4058. if !bytes.Equal(buf2, buf[offset:offset+512]) {
  4059. logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
  4060. return
  4061. }
  4062. offset += 512
  4063. m, err = r.ReadAt(buf3, offset)
  4064. if err != nil {
  4065. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  4066. return
  4067. }
  4068. if m != len(buf3) {
  4069. logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err)
  4070. return
  4071. }
  4072. if !bytes.Equal(buf3, buf[offset:offset+512]) {
  4073. logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
  4074. return
  4075. }
  4076. offset += 512
  4077. m, err = r.ReadAt(buf4, offset)
  4078. if err != nil {
  4079. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  4080. return
  4081. }
  4082. if m != len(buf4) {
  4083. logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err)
  4084. return
  4085. }
  4086. if !bytes.Equal(buf4, buf[offset:offset+512]) {
  4087. logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
  4088. return
  4089. }
  4090. buf5 := make([]byte, len(buf))
  4091. // Read the whole object.
  4092. m, err = r.ReadAt(buf5, 0)
  4093. if err != nil {
  4094. if err != io.EOF {
  4095. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  4096. return
  4097. }
  4098. }
  4099. if m != len(buf5) {
  4100. logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err)
  4101. return
  4102. }
  4103. if !bytes.Equal(buf, buf5) {
  4104. logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err)
  4105. return
  4106. }
  4107. buf6 := make([]byte, len(buf)+1)
  4108. // Read the whole object and beyond.
  4109. _, err = r.ReadAt(buf6, 0)
  4110. if err != nil {
  4111. if err != io.EOF {
  4112. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  4113. return
  4114. }
  4115. }
  4116. successLogger(testName, function, args, startTime).Info()
  4117. }
  4118. // testSSECEncryptionPutGet tests encryption with customer provided encryption keys
  4119. func testSSECEncryptionPutGet() {
  4120. // initialize logging params
  4121. startTime := time.Now()
  4122. testName := getFuncName()
  4123. function := "PutEncryptedObject(bucketName, objectName, reader, sse)"
  4124. args := map[string]interface{}{
  4125. "bucketName": "",
  4126. "objectName": "",
  4127. "sse": "",
  4128. }
  4129. // Seed random based on current time.
  4130. rand.Seed(time.Now().Unix())
  4131. // Instantiate new minio client object
  4132. c, err := minio.New(os.Getenv(serverEndpoint),
  4133. &minio.Options{
  4134. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  4135. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  4136. })
  4137. if err != nil {
  4138. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  4139. return
  4140. }
  4141. // Enable tracing, write to stderr.
  4142. // c.TraceOn(os.Stderr)
  4143. // Set user agent.
  4144. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  4145. // Generate a new random bucket name.
  4146. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  4147. args["bucketName"] = bucketName
  4148. // Make a new bucket.
  4149. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  4150. if err != nil {
  4151. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  4152. return
  4153. }
  4154. defer cleanupBucket(bucketName, c)
  4155. testCases := []struct {
  4156. buf []byte
  4157. }{
  4158. {buf: bytes.Repeat([]byte("F"), 1)},
  4159. {buf: bytes.Repeat([]byte("F"), 15)},
  4160. {buf: bytes.Repeat([]byte("F"), 16)},
  4161. {buf: bytes.Repeat([]byte("F"), 17)},
  4162. {buf: bytes.Repeat([]byte("F"), 31)},
  4163. {buf: bytes.Repeat([]byte("F"), 32)},
  4164. {buf: bytes.Repeat([]byte("F"), 33)},
  4165. {buf: bytes.Repeat([]byte("F"), 1024)},
  4166. {buf: bytes.Repeat([]byte("F"), 1024*2)},
  4167. {buf: bytes.Repeat([]byte("F"), 1024*1024)},
  4168. }
  4169. const password = "correct horse battery staple" // https://xkcd.com/936/
  4170. for i, testCase := range testCases {
  4171. // Generate a random object name
  4172. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  4173. args["objectName"] = objectName
  4174. // Secured object
  4175. sse := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName))
  4176. args["sse"] = sse
  4177. // Put encrypted data
  4178. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(testCase.buf), int64(len(testCase.buf)), minio.PutObjectOptions{ServerSideEncryption: sse})
  4179. if err != nil {
  4180. logError(testName, function, args, startTime, "", "PutEncryptedObject failed", err)
  4181. return
  4182. }
  4183. // Read the data back
  4184. r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse})
  4185. if err != nil {
  4186. logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err)
  4187. return
  4188. }
  4189. defer r.Close()
  4190. // Compare the sent object with the received one
  4191. recvBuffer := bytes.NewBuffer([]byte{})
  4192. if _, err = io.Copy(recvBuffer, r); err != nil {
  4193. logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err)
  4194. return
  4195. }
  4196. if recvBuffer.Len() != len(testCase.buf) {
  4197. logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err)
  4198. return
  4199. }
  4200. if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) {
  4201. logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err)
  4202. return
  4203. }
  4204. successLogger(testName, function, args, startTime).Info()
  4205. }
  4206. successLogger(testName, function, args, startTime).Info()
  4207. }
  4208. // TestEncryptionFPut tests encryption with customer specified encryption keys
  4209. func testSSECEncryptionFPut() {
  4210. // initialize logging params
  4211. startTime := time.Now()
  4212. testName := getFuncName()
  4213. function := "FPutEncryptedObject(bucketName, objectName, filePath, contentType, sse)"
  4214. args := map[string]interface{}{
  4215. "bucketName": "",
  4216. "objectName": "",
  4217. "filePath": "",
  4218. "contentType": "",
  4219. "sse": "",
  4220. }
  4221. // Seed random based on current time.
  4222. rand.Seed(time.Now().Unix())
  4223. // Instantiate new minio client object
  4224. c, err := minio.New(os.Getenv(serverEndpoint),
  4225. &minio.Options{
  4226. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  4227. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  4228. })
  4229. if err != nil {
  4230. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  4231. return
  4232. }
  4233. // Enable tracing, write to stderr.
  4234. // c.TraceOn(os.Stderr)
  4235. // Set user agent.
  4236. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  4237. // Generate a new random bucket name.
  4238. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  4239. args["bucketName"] = bucketName
  4240. // Make a new bucket.
  4241. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  4242. if err != nil {
  4243. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  4244. return
  4245. }
  4246. defer cleanupBucket(bucketName, c)
  4247. // Object custom metadata
  4248. customContentType := "custom/contenttype"
  4249. args["metadata"] = customContentType
  4250. testCases := []struct {
  4251. buf []byte
  4252. }{
  4253. {buf: bytes.Repeat([]byte("F"), 0)},
  4254. {buf: bytes.Repeat([]byte("F"), 1)},
  4255. {buf: bytes.Repeat([]byte("F"), 15)},
  4256. {buf: bytes.Repeat([]byte("F"), 16)},
  4257. {buf: bytes.Repeat([]byte("F"), 17)},
  4258. {buf: bytes.Repeat([]byte("F"), 31)},
  4259. {buf: bytes.Repeat([]byte("F"), 32)},
  4260. {buf: bytes.Repeat([]byte("F"), 33)},
  4261. {buf: bytes.Repeat([]byte("F"), 1024)},
  4262. {buf: bytes.Repeat([]byte("F"), 1024*2)},
  4263. {buf: bytes.Repeat([]byte("F"), 1024*1024)},
  4264. }
  4265. const password = "correct horse battery staple" // https://xkcd.com/936/
  4266. for i, testCase := range testCases {
  4267. // Generate a random object name
  4268. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  4269. args["objectName"] = objectName
  4270. // Secured object
  4271. sse := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName))
  4272. args["sse"] = sse
  4273. // Generate a random file name.
  4274. fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  4275. file, err := os.Create(fileName)
  4276. if err != nil {
  4277. logError(testName, function, args, startTime, "", "file create failed", err)
  4278. return
  4279. }
  4280. _, err = file.Write(testCase.buf)
  4281. if err != nil {
  4282. logError(testName, function, args, startTime, "", "file write failed", err)
  4283. return
  4284. }
  4285. file.Close()
  4286. // Put encrypted data
  4287. if _, err = c.FPutObject(context.Background(), bucketName, objectName, fileName, minio.PutObjectOptions{ServerSideEncryption: sse}); err != nil {
  4288. logError(testName, function, args, startTime, "", "FPutEncryptedObject failed", err)
  4289. return
  4290. }
  4291. // Read the data back
  4292. r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse})
  4293. if err != nil {
  4294. logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err)
  4295. return
  4296. }
  4297. defer r.Close()
  4298. // Compare the sent object with the received one
  4299. recvBuffer := bytes.NewBuffer([]byte{})
  4300. if _, err = io.Copy(recvBuffer, r); err != nil {
  4301. logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err)
  4302. return
  4303. }
  4304. if recvBuffer.Len() != len(testCase.buf) {
  4305. logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err)
  4306. return
  4307. }
  4308. if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) {
  4309. logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err)
  4310. return
  4311. }
  4312. os.Remove(fileName)
  4313. }
  4314. successLogger(testName, function, args, startTime).Info()
  4315. }
  4316. // testSSES3EncryptionPutGet tests SSE-S3 encryption
  4317. func testSSES3EncryptionPutGet() {
  4318. // initialize logging params
  4319. startTime := time.Now()
  4320. testName := getFuncName()
  4321. function := "PutEncryptedObject(bucketName, objectName, reader, sse)"
  4322. args := map[string]interface{}{
  4323. "bucketName": "",
  4324. "objectName": "",
  4325. "sse": "",
  4326. }
  4327. // Seed random based on current time.
  4328. rand.Seed(time.Now().Unix())
  4329. // Instantiate new minio client object
  4330. c, err := minio.New(os.Getenv(serverEndpoint),
  4331. &minio.Options{
  4332. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  4333. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  4334. })
  4335. if err != nil {
  4336. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  4337. return
  4338. }
  4339. // Enable tracing, write to stderr.
  4340. // c.TraceOn(os.Stderr)
  4341. // Set user agent.
  4342. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  4343. // Generate a new random bucket name.
  4344. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  4345. args["bucketName"] = bucketName
  4346. // Make a new bucket.
  4347. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  4348. if err != nil {
  4349. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  4350. return
  4351. }
  4352. defer cleanupBucket(bucketName, c)
  4353. testCases := []struct {
  4354. buf []byte
  4355. }{
  4356. {buf: bytes.Repeat([]byte("F"), 1)},
  4357. {buf: bytes.Repeat([]byte("F"), 15)},
  4358. {buf: bytes.Repeat([]byte("F"), 16)},
  4359. {buf: bytes.Repeat([]byte("F"), 17)},
  4360. {buf: bytes.Repeat([]byte("F"), 31)},
  4361. {buf: bytes.Repeat([]byte("F"), 32)},
  4362. {buf: bytes.Repeat([]byte("F"), 33)},
  4363. {buf: bytes.Repeat([]byte("F"), 1024)},
  4364. {buf: bytes.Repeat([]byte("F"), 1024*2)},
  4365. {buf: bytes.Repeat([]byte("F"), 1024*1024)},
  4366. }
  4367. for i, testCase := range testCases {
  4368. // Generate a random object name
  4369. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  4370. args["objectName"] = objectName
  4371. // Secured object
  4372. sse := encrypt.NewSSE()
  4373. args["sse"] = sse
  4374. // Put encrypted data
  4375. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(testCase.buf), int64(len(testCase.buf)), minio.PutObjectOptions{ServerSideEncryption: sse})
  4376. if err != nil {
  4377. logError(testName, function, args, startTime, "", "PutEncryptedObject failed", err)
  4378. return
  4379. }
  4380. // Read the data back without any encryption headers
  4381. r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  4382. if err != nil {
  4383. logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err)
  4384. return
  4385. }
  4386. defer r.Close()
  4387. // Compare the sent object with the received one
  4388. recvBuffer := bytes.NewBuffer([]byte{})
  4389. if _, err = io.Copy(recvBuffer, r); err != nil {
  4390. logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err)
  4391. return
  4392. }
  4393. if recvBuffer.Len() != len(testCase.buf) {
  4394. logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err)
  4395. return
  4396. }
  4397. if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) {
  4398. logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err)
  4399. return
  4400. }
  4401. successLogger(testName, function, args, startTime).Info()
  4402. }
  4403. successLogger(testName, function, args, startTime).Info()
  4404. }
  4405. // TestSSES3EncryptionFPut tests server side encryption
  4406. func testSSES3EncryptionFPut() {
  4407. // initialize logging params
  4408. startTime := time.Now()
  4409. testName := getFuncName()
  4410. function := "FPutEncryptedObject(bucketName, objectName, filePath, contentType, sse)"
  4411. args := map[string]interface{}{
  4412. "bucketName": "",
  4413. "objectName": "",
  4414. "filePath": "",
  4415. "contentType": "",
  4416. "sse": "",
  4417. }
  4418. // Seed random based on current time.
  4419. rand.Seed(time.Now().Unix())
  4420. // Instantiate new minio client object
  4421. c, err := minio.New(os.Getenv(serverEndpoint),
  4422. &minio.Options{
  4423. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  4424. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  4425. })
  4426. if err != nil {
  4427. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  4428. return
  4429. }
  4430. // Enable tracing, write to stderr.
  4431. // c.TraceOn(os.Stderr)
  4432. // Set user agent.
  4433. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  4434. // Generate a new random bucket name.
  4435. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  4436. args["bucketName"] = bucketName
  4437. // Make a new bucket.
  4438. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  4439. if err != nil {
  4440. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  4441. return
  4442. }
  4443. defer cleanupBucket(bucketName, c)
  4444. // Object custom metadata
  4445. customContentType := "custom/contenttype"
  4446. args["metadata"] = customContentType
  4447. testCases := []struct {
  4448. buf []byte
  4449. }{
  4450. {buf: bytes.Repeat([]byte("F"), 0)},
  4451. {buf: bytes.Repeat([]byte("F"), 1)},
  4452. {buf: bytes.Repeat([]byte("F"), 15)},
  4453. {buf: bytes.Repeat([]byte("F"), 16)},
  4454. {buf: bytes.Repeat([]byte("F"), 17)},
  4455. {buf: bytes.Repeat([]byte("F"), 31)},
  4456. {buf: bytes.Repeat([]byte("F"), 32)},
  4457. {buf: bytes.Repeat([]byte("F"), 33)},
  4458. {buf: bytes.Repeat([]byte("F"), 1024)},
  4459. {buf: bytes.Repeat([]byte("F"), 1024*2)},
  4460. {buf: bytes.Repeat([]byte("F"), 1024*1024)},
  4461. }
  4462. for i, testCase := range testCases {
  4463. // Generate a random object name
  4464. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  4465. args["objectName"] = objectName
  4466. // Secured object
  4467. sse := encrypt.NewSSE()
  4468. args["sse"] = sse
  4469. // Generate a random file name.
  4470. fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  4471. file, err := os.Create(fileName)
  4472. if err != nil {
  4473. logError(testName, function, args, startTime, "", "file create failed", err)
  4474. return
  4475. }
  4476. _, err = file.Write(testCase.buf)
  4477. if err != nil {
  4478. logError(testName, function, args, startTime, "", "file write failed", err)
  4479. return
  4480. }
  4481. file.Close()
  4482. // Put encrypted data
  4483. if _, err = c.FPutObject(context.Background(), bucketName, objectName, fileName, minio.PutObjectOptions{ServerSideEncryption: sse}); err != nil {
  4484. logError(testName, function, args, startTime, "", "FPutEncryptedObject failed", err)
  4485. return
  4486. }
  4487. // Read the data back
  4488. r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  4489. if err != nil {
  4490. logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err)
  4491. return
  4492. }
  4493. defer r.Close()
  4494. // Compare the sent object with the received one
  4495. recvBuffer := bytes.NewBuffer([]byte{})
  4496. if _, err = io.Copy(recvBuffer, r); err != nil {
  4497. logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err)
  4498. return
  4499. }
  4500. if recvBuffer.Len() != len(testCase.buf) {
  4501. logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err)
  4502. return
  4503. }
  4504. if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) {
  4505. logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err)
  4506. return
  4507. }
  4508. os.Remove(fileName)
  4509. }
  4510. successLogger(testName, function, args, startTime).Info()
  4511. }
  4512. func testBucketNotification() {
  4513. // initialize logging params
  4514. startTime := time.Now()
  4515. testName := getFuncName()
  4516. function := "SetBucketNotification(bucketName)"
  4517. args := map[string]interface{}{
  4518. "bucketName": "",
  4519. }
  4520. if os.Getenv("NOTIFY_BUCKET") == "" ||
  4521. os.Getenv("NOTIFY_SERVICE") == "" ||
  4522. os.Getenv("NOTIFY_REGION") == "" ||
  4523. os.Getenv("NOTIFY_ACCOUNTID") == "" ||
  4524. os.Getenv("NOTIFY_RESOURCE") == "" {
  4525. ignoredLog(testName, function, args, startTime, "Skipped notification test as it is not configured").Info()
  4526. return
  4527. }
  4528. // Seed random based on current time.
  4529. rand.Seed(time.Now().Unix())
  4530. c, err := minio.New(os.Getenv(serverEndpoint),
  4531. &minio.Options{
  4532. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  4533. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  4534. })
  4535. if err != nil {
  4536. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  4537. return
  4538. }
  4539. // Enable to debug
  4540. // c.TraceOn(os.Stderr)
  4541. // Set user agent.
  4542. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  4543. bucketName := os.Getenv("NOTIFY_BUCKET")
  4544. args["bucketName"] = bucketName
  4545. topicArn := notification.NewArn("aws", os.Getenv("NOTIFY_SERVICE"), os.Getenv("NOTIFY_REGION"), os.Getenv("NOTIFY_ACCOUNTID"), os.Getenv("NOTIFY_RESOURCE"))
  4546. queueArn := notification.NewArn("aws", "dummy-service", "dummy-region", "dummy-accountid", "dummy-resource")
  4547. topicConfig := notification.NewConfig(topicArn)
  4548. topicConfig.AddEvents(notification.ObjectCreatedAll, notification.ObjectRemovedAll)
  4549. topicConfig.AddFilterSuffix("jpg")
  4550. queueConfig := notification.NewConfig(queueArn)
  4551. queueConfig.AddEvents(notification.ObjectCreatedAll)
  4552. queueConfig.AddFilterPrefix("photos/")
  4553. config := notification.Configuration{}
  4554. config.AddTopic(topicConfig)
  4555. // Add the same topicConfig again, should have no effect
  4556. // because it is duplicated
  4557. config.AddTopic(topicConfig)
  4558. if len(config.TopicConfigs) != 1 {
  4559. logError(testName, function, args, startTime, "", "Duplicate entry added", err)
  4560. return
  4561. }
  4562. // Add and remove a queue config
  4563. config.AddQueue(queueConfig)
  4564. config.RemoveQueueByArn(queueArn)
  4565. err = c.SetBucketNotification(context.Background(), bucketName, config)
  4566. if err != nil {
  4567. logError(testName, function, args, startTime, "", "SetBucketNotification failed", err)
  4568. return
  4569. }
  4570. config, err = c.GetBucketNotification(context.Background(), bucketName)
  4571. if err != nil {
  4572. logError(testName, function, args, startTime, "", "GetBucketNotification failed", err)
  4573. return
  4574. }
  4575. if len(config.TopicConfigs) != 1 {
  4576. logError(testName, function, args, startTime, "", "Topic config is empty", err)
  4577. return
  4578. }
  4579. if config.TopicConfigs[0].Filter.S3Key.FilterRules[0].Value != "jpg" {
  4580. logError(testName, function, args, startTime, "", "Couldn't get the suffix", err)
  4581. return
  4582. }
  4583. err = c.RemoveAllBucketNotification(context.Background(), bucketName)
  4584. if err != nil {
  4585. logError(testName, function, args, startTime, "", "RemoveAllBucketNotification failed", err)
  4586. return
  4587. }
  4588. // Delete all objects and buckets
  4589. if err = cleanupBucket(bucketName, c); err != nil {
  4590. logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
  4591. return
  4592. }
  4593. successLogger(testName, function, args, startTime).Info()
  4594. }
  4595. // Tests comprehensive list of all methods.
  4596. func testFunctional() {
  4597. // initialize logging params
  4598. startTime := time.Now()
  4599. testName := getFuncName()
  4600. function := "testFunctional()"
  4601. functionAll := ""
  4602. args := map[string]interface{}{}
  4603. // Seed random based on current time.
  4604. rand.Seed(time.Now().Unix())
  4605. c, err := minio.New(os.Getenv(serverEndpoint),
  4606. &minio.Options{
  4607. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  4608. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  4609. })
  4610. if err != nil {
  4611. logError(testName, function, nil, startTime, "", "MinIO client object creation failed", err)
  4612. return
  4613. }
  4614. // Enable to debug
  4615. // c.TraceOn(os.Stderr)
  4616. // Set user agent.
  4617. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  4618. // Generate a new random bucket name.
  4619. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  4620. // Make a new bucket.
  4621. function = "MakeBucket(bucketName, region)"
  4622. functionAll = "MakeBucket(bucketName, region)"
  4623. args["bucketName"] = bucketName
  4624. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  4625. defer cleanupBucket(bucketName, c)
  4626. if err != nil {
  4627. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  4628. return
  4629. }
  4630. // Generate a random file name.
  4631. fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  4632. file, err := os.Create(fileName)
  4633. if err != nil {
  4634. logError(testName, function, args, startTime, "", "File creation failed", err)
  4635. return
  4636. }
  4637. for i := 0; i < 3; i++ {
  4638. buf := make([]byte, rand.Intn(1<<19))
  4639. _, err = file.Write(buf)
  4640. if err != nil {
  4641. logError(testName, function, args, startTime, "", "File write failed", err)
  4642. return
  4643. }
  4644. }
  4645. file.Close()
  4646. // Verify if bucket exits and you have access.
  4647. var exists bool
  4648. function = "BucketExists(bucketName)"
  4649. functionAll += ", " + function
  4650. args = map[string]interface{}{
  4651. "bucketName": bucketName,
  4652. }
  4653. exists, err = c.BucketExists(context.Background(), bucketName)
  4654. if err != nil {
  4655. logError(testName, function, args, startTime, "", "BucketExists failed", err)
  4656. return
  4657. }
  4658. if !exists {
  4659. logError(testName, function, args, startTime, "", "Could not find the bucket", err)
  4660. return
  4661. }
  4662. // Asserting the default bucket policy.
  4663. function = "GetBucketPolicy(ctx, bucketName)"
  4664. functionAll += ", " + function
  4665. args = map[string]interface{}{
  4666. "bucketName": bucketName,
  4667. }
  4668. nilPolicy, err := c.GetBucketPolicy(context.Background(), bucketName)
  4669. if err != nil {
  4670. logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err)
  4671. return
  4672. }
  4673. if nilPolicy != "" {
  4674. logError(testName, function, args, startTime, "", "policy should be set to nil", err)
  4675. return
  4676. }
  4677. // Set the bucket policy to 'public readonly'.
  4678. function = "SetBucketPolicy(bucketName, readOnlyPolicy)"
  4679. functionAll += ", " + function
  4680. readOnlyPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}`
  4681. args = map[string]interface{}{
  4682. "bucketName": bucketName,
  4683. "bucketPolicy": readOnlyPolicy,
  4684. }
  4685. err = c.SetBucketPolicy(context.Background(), bucketName, readOnlyPolicy)
  4686. if err != nil {
  4687. logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err)
  4688. return
  4689. }
  4690. // should return policy `readonly`.
  4691. function = "GetBucketPolicy(ctx, bucketName)"
  4692. functionAll += ", " + function
  4693. args = map[string]interface{}{
  4694. "bucketName": bucketName,
  4695. }
  4696. _, err = c.GetBucketPolicy(context.Background(), bucketName)
  4697. if err != nil {
  4698. logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err)
  4699. return
  4700. }
  4701. // Make the bucket 'public writeonly'.
  4702. function = "SetBucketPolicy(bucketName, writeOnlyPolicy)"
  4703. functionAll += ", " + function
  4704. writeOnlyPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}`
  4705. args = map[string]interface{}{
  4706. "bucketName": bucketName,
  4707. "bucketPolicy": writeOnlyPolicy,
  4708. }
  4709. err = c.SetBucketPolicy(context.Background(), bucketName, writeOnlyPolicy)
  4710. if err != nil {
  4711. logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err)
  4712. return
  4713. }
  4714. // should return policy `writeonly`.
  4715. function = "GetBucketPolicy(ctx, bucketName)"
  4716. functionAll += ", " + function
  4717. args = map[string]interface{}{
  4718. "bucketName": bucketName,
  4719. }
  4720. _, err = c.GetBucketPolicy(context.Background(), bucketName)
  4721. if err != nil {
  4722. logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err)
  4723. return
  4724. }
  4725. // Make the bucket 'public read/write'.
  4726. function = "SetBucketPolicy(bucketName, readWritePolicy)"
  4727. functionAll += ", " + function
  4728. readWritePolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}`
  4729. args = map[string]interface{}{
  4730. "bucketName": bucketName,
  4731. "bucketPolicy": readWritePolicy,
  4732. }
  4733. err = c.SetBucketPolicy(context.Background(), bucketName, readWritePolicy)
  4734. if err != nil {
  4735. logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err)
  4736. return
  4737. }
  4738. // should return policy `readwrite`.
  4739. function = "GetBucketPolicy(bucketName)"
  4740. functionAll += ", " + function
  4741. args = map[string]interface{}{
  4742. "bucketName": bucketName,
  4743. }
  4744. _, err = c.GetBucketPolicy(context.Background(), bucketName)
  4745. if err != nil {
  4746. logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err)
  4747. return
  4748. }
  4749. // List all buckets.
  4750. function = "ListBuckets()"
  4751. functionAll += ", " + function
  4752. args = nil
  4753. buckets, err := c.ListBuckets(context.Background())
  4754. if len(buckets) == 0 {
  4755. logError(testName, function, args, startTime, "", "Found bucket list to be empty", err)
  4756. return
  4757. }
  4758. if err != nil {
  4759. logError(testName, function, args, startTime, "", "ListBuckets failed", err)
  4760. return
  4761. }
  4762. // Verify if previously created bucket is listed in list buckets.
  4763. bucketFound := false
  4764. for _, bucket := range buckets {
  4765. if bucket.Name == bucketName {
  4766. bucketFound = true
  4767. }
  4768. }
  4769. // If bucket not found error out.
  4770. if !bucketFound {
  4771. logError(testName, function, args, startTime, "", "Bucket: "+bucketName+" not found", err)
  4772. return
  4773. }
  4774. objectName := bucketName + "unique"
  4775. // Generate data
  4776. buf := bytes.Repeat([]byte("f"), 1<<19)
  4777. function = "PutObject(bucketName, objectName, reader, contentType)"
  4778. functionAll += ", " + function
  4779. args = map[string]interface{}{
  4780. "bucketName": bucketName,
  4781. "objectName": objectName,
  4782. "contentType": "",
  4783. }
  4784. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{})
  4785. if err != nil {
  4786. logError(testName, function, args, startTime, "", "PutObject failed", err)
  4787. return
  4788. }
  4789. args = map[string]interface{}{
  4790. "bucketName": bucketName,
  4791. "objectName": objectName + "-nolength",
  4792. "contentType": "binary/octet-stream",
  4793. }
  4794. _, err = c.PutObject(context.Background(), bucketName, objectName+"-nolength", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  4795. if err != nil {
  4796. logError(testName, function, args, startTime, "", "PutObject failed", err)
  4797. return
  4798. }
  4799. // Instantiate a done channel to close all listing.
  4800. doneCh := make(chan struct{})
  4801. defer close(doneCh)
  4802. objFound := false
  4803. isRecursive := true // Recursive is true.
  4804. function = "ListObjects(bucketName, objectName, isRecursive, doneCh)"
  4805. functionAll += ", " + function
  4806. args = map[string]interface{}{
  4807. "bucketName": bucketName,
  4808. "objectName": objectName,
  4809. "isRecursive": isRecursive,
  4810. }
  4811. for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Prefix: objectName, Recursive: true}) {
  4812. if obj.Key == objectName {
  4813. objFound = true
  4814. break
  4815. }
  4816. }
  4817. if !objFound {
  4818. logError(testName, function, args, startTime, "", "Object "+objectName+" not found", err)
  4819. return
  4820. }
  4821. objFound = false
  4822. isRecursive = true // Recursive is true.
  4823. function = "ListObjects()"
  4824. functionAll += ", " + function
  4825. args = map[string]interface{}{
  4826. "bucketName": bucketName,
  4827. "objectName": objectName,
  4828. "isRecursive": isRecursive,
  4829. }
  4830. for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{Prefix: objectName, Recursive: isRecursive}) {
  4831. if obj.Key == objectName {
  4832. objFound = true
  4833. break
  4834. }
  4835. }
  4836. if !objFound {
  4837. logError(testName, function, args, startTime, "", "Object "+objectName+" not found", err)
  4838. return
  4839. }
  4840. incompObjNotFound := true
  4841. function = "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)"
  4842. functionAll += ", " + function
  4843. args = map[string]interface{}{
  4844. "bucketName": bucketName,
  4845. "objectName": objectName,
  4846. "isRecursive": isRecursive,
  4847. }
  4848. for objIncompl := range c.ListIncompleteUploads(context.Background(), bucketName, objectName, isRecursive) {
  4849. if objIncompl.Key != "" {
  4850. incompObjNotFound = false
  4851. break
  4852. }
  4853. }
  4854. if !incompObjNotFound {
  4855. logError(testName, function, args, startTime, "", "Unexpected dangling incomplete upload found", err)
  4856. return
  4857. }
  4858. function = "GetObject(bucketName, objectName)"
  4859. functionAll += ", " + function
  4860. args = map[string]interface{}{
  4861. "bucketName": bucketName,
  4862. "objectName": objectName,
  4863. }
  4864. newReader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  4865. if err != nil {
  4866. logError(testName, function, args, startTime, "", "GetObject failed", err)
  4867. return
  4868. }
  4869. newReadBytes, err := ioutil.ReadAll(newReader)
  4870. if err != nil {
  4871. logError(testName, function, args, startTime, "", "ReadAll failed", err)
  4872. return
  4873. }
  4874. if !bytes.Equal(newReadBytes, buf) {
  4875. logError(testName, function, args, startTime, "", "GetObject bytes mismatch", err)
  4876. return
  4877. }
  4878. newReader.Close()
  4879. function = "FGetObject(bucketName, objectName, fileName)"
  4880. functionAll += ", " + function
  4881. args = map[string]interface{}{
  4882. "bucketName": bucketName,
  4883. "objectName": objectName,
  4884. "fileName": fileName + "-f",
  4885. }
  4886. err = c.FGetObject(context.Background(), bucketName, objectName, fileName+"-f", minio.GetObjectOptions{})
  4887. if err != nil {
  4888. logError(testName, function, args, startTime, "", "FGetObject failed", err)
  4889. return
  4890. }
  4891. function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)"
  4892. functionAll += ", " + function
  4893. args = map[string]interface{}{
  4894. "bucketName": bucketName,
  4895. "objectName": "",
  4896. "expires": 3600 * time.Second,
  4897. }
  4898. if _, err = c.PresignedHeadObject(context.Background(), bucketName, "", 3600*time.Second, nil); err == nil {
  4899. logError(testName, function, args, startTime, "", "PresignedHeadObject success", err)
  4900. return
  4901. }
  4902. // Generate presigned HEAD object url.
  4903. function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)"
  4904. functionAll += ", " + function
  4905. args = map[string]interface{}{
  4906. "bucketName": bucketName,
  4907. "objectName": objectName,
  4908. "expires": 3600 * time.Second,
  4909. }
  4910. presignedHeadURL, err := c.PresignedHeadObject(context.Background(), bucketName, objectName, 3600*time.Second, nil)
  4911. if err != nil {
  4912. logError(testName, function, args, startTime, "", "PresignedHeadObject failed", err)
  4913. return
  4914. }
  4915. transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS)))
  4916. if err != nil {
  4917. logError(testName, function, args, startTime, "", "DefaultTransport failed", err)
  4918. return
  4919. }
  4920. httpClient := &http.Client{
  4921. // Setting a sensible time out of 30secs to wait for response
  4922. // headers. Request is pro-actively canceled after 30secs
  4923. // with no response.
  4924. Timeout: 30 * time.Second,
  4925. Transport: transport,
  4926. }
  4927. req, err := http.NewRequest(http.MethodHead, presignedHeadURL.String(), nil)
  4928. if err != nil {
  4929. logError(testName, function, args, startTime, "", "PresignedHeadObject request was incorrect", err)
  4930. return
  4931. }
  4932. // Verify if presigned url works.
  4933. resp, err := httpClient.Do(req)
  4934. if err != nil {
  4935. logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect", err)
  4936. return
  4937. }
  4938. if resp.StatusCode != http.StatusOK {
  4939. logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect, status "+string(resp.StatusCode), err)
  4940. return
  4941. }
  4942. if resp.Header.Get("ETag") == "" {
  4943. logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect", err)
  4944. return
  4945. }
  4946. resp.Body.Close()
  4947. function = "PresignedGetObject(bucketName, objectName, expires, reqParams)"
  4948. functionAll += ", " + function
  4949. args = map[string]interface{}{
  4950. "bucketName": bucketName,
  4951. "objectName": "",
  4952. "expires": 3600 * time.Second,
  4953. }
  4954. _, err = c.PresignedGetObject(context.Background(), bucketName, "", 3600*time.Second, nil)
  4955. if err == nil {
  4956. logError(testName, function, args, startTime, "", "PresignedGetObject success", err)
  4957. return
  4958. }
  4959. // Generate presigned GET object url.
  4960. function = "PresignedGetObject(bucketName, objectName, expires, reqParams)"
  4961. functionAll += ", " + function
  4962. args = map[string]interface{}{
  4963. "bucketName": bucketName,
  4964. "objectName": objectName,
  4965. "expires": 3600 * time.Second,
  4966. }
  4967. presignedGetURL, err := c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, nil)
  4968. if err != nil {
  4969. logError(testName, function, args, startTime, "", "PresignedGetObject failed", err)
  4970. return
  4971. }
  4972. // Verify if presigned url works.
  4973. req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil)
  4974. if err != nil {
  4975. logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err)
  4976. return
  4977. }
  4978. resp, err = httpClient.Do(req)
  4979. if err != nil {
  4980. logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
  4981. return
  4982. }
  4983. if resp.StatusCode != http.StatusOK {
  4984. logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err)
  4985. return
  4986. }
  4987. newPresignedBytes, err := ioutil.ReadAll(resp.Body)
  4988. if err != nil {
  4989. logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
  4990. return
  4991. }
  4992. resp.Body.Close()
  4993. if !bytes.Equal(newPresignedBytes, buf) {
  4994. logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
  4995. return
  4996. }
  4997. // Set request parameters.
  4998. reqParams := make(url.Values)
  4999. reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"")
  5000. args = map[string]interface{}{
  5001. "bucketName": bucketName,
  5002. "objectName": objectName,
  5003. "expires": 3600 * time.Second,
  5004. "reqParams": reqParams,
  5005. }
  5006. presignedGetURL, err = c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, reqParams)
  5007. if err != nil {
  5008. logError(testName, function, args, startTime, "", "PresignedGetObject failed", err)
  5009. return
  5010. }
  5011. // Verify if presigned url works.
  5012. req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil)
  5013. if err != nil {
  5014. logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err)
  5015. return
  5016. }
  5017. resp, err = httpClient.Do(req)
  5018. if err != nil {
  5019. logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
  5020. return
  5021. }
  5022. if resp.StatusCode != http.StatusOK {
  5023. logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err)
  5024. return
  5025. }
  5026. newPresignedBytes, err = ioutil.ReadAll(resp.Body)
  5027. if err != nil {
  5028. logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
  5029. return
  5030. }
  5031. if !bytes.Equal(newPresignedBytes, buf) {
  5032. logError(testName, function, args, startTime, "", "Bytes mismatch for presigned GET URL", err)
  5033. return
  5034. }
  5035. if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" {
  5036. logError(testName, function, args, startTime, "", "wrong Content-Disposition received "+string(resp.Header.Get("Content-Disposition")), err)
  5037. return
  5038. }
  5039. function = "PresignedPutObject(bucketName, objectName, expires)"
  5040. functionAll += ", " + function
  5041. args = map[string]interface{}{
  5042. "bucketName": bucketName,
  5043. "objectName": "",
  5044. "expires": 3600 * time.Second,
  5045. }
  5046. _, err = c.PresignedPutObject(context.Background(), bucketName, "", 3600*time.Second)
  5047. if err == nil {
  5048. logError(testName, function, args, startTime, "", "PresignedPutObject success", err)
  5049. return
  5050. }
  5051. function = "PresignedPutObject(bucketName, objectName, expires)"
  5052. functionAll += ", " + function
  5053. args = map[string]interface{}{
  5054. "bucketName": bucketName,
  5055. "objectName": objectName + "-presigned",
  5056. "expires": 3600 * time.Second,
  5057. }
  5058. presignedPutURL, err := c.PresignedPutObject(context.Background(), bucketName, objectName+"-presigned", 3600*time.Second)
  5059. if err != nil {
  5060. logError(testName, function, args, startTime, "", "PresignedPutObject failed", err)
  5061. return
  5062. }
  5063. buf = bytes.Repeat([]byte("g"), 1<<19)
  5064. req, err = http.NewRequest(http.MethodPut, presignedPutURL.String(), bytes.NewReader(buf))
  5065. if err != nil {
  5066. logError(testName, function, args, startTime, "", "Couldn't make HTTP request with PresignedPutObject URL", err)
  5067. return
  5068. }
  5069. resp, err = httpClient.Do(req)
  5070. if err != nil {
  5071. logError(testName, function, args, startTime, "", "PresignedPutObject failed", err)
  5072. return
  5073. }
  5074. newReader, err = c.GetObject(context.Background(), bucketName, objectName+"-presigned", minio.GetObjectOptions{})
  5075. if err != nil {
  5076. logError(testName, function, args, startTime, "", "GetObject after PresignedPutObject failed", err)
  5077. return
  5078. }
  5079. newReadBytes, err = ioutil.ReadAll(newReader)
  5080. if err != nil {
  5081. logError(testName, function, args, startTime, "", "ReadAll after GetObject failed", err)
  5082. return
  5083. }
  5084. if !bytes.Equal(newReadBytes, buf) {
  5085. logError(testName, function, args, startTime, "", "Bytes mismatch", err)
  5086. return
  5087. }
  5088. function = "RemoveObject(bucketName, objectName)"
  5089. functionAll += ", " + function
  5090. args = map[string]interface{}{
  5091. "bucketName": bucketName,
  5092. "objectName": objectName,
  5093. }
  5094. err = c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{})
  5095. if err != nil {
  5096. logError(testName, function, args, startTime, "", "RemoveObject failed", err)
  5097. return
  5098. }
  5099. args["objectName"] = objectName + "-f"
  5100. err = c.RemoveObject(context.Background(), bucketName, objectName+"-f", minio.RemoveObjectOptions{})
  5101. if err != nil {
  5102. logError(testName, function, args, startTime, "", "RemoveObject failed", err)
  5103. return
  5104. }
  5105. args["objectName"] = objectName + "-nolength"
  5106. err = c.RemoveObject(context.Background(), bucketName, objectName+"-nolength", minio.RemoveObjectOptions{})
  5107. if err != nil {
  5108. logError(testName, function, args, startTime, "", "RemoveObject failed", err)
  5109. return
  5110. }
  5111. args["objectName"] = objectName + "-presigned"
  5112. err = c.RemoveObject(context.Background(), bucketName, objectName+"-presigned", minio.RemoveObjectOptions{})
  5113. if err != nil {
  5114. logError(testName, function, args, startTime, "", "RemoveObject failed", err)
  5115. return
  5116. }
  5117. function = "RemoveBucket(bucketName)"
  5118. functionAll += ", " + function
  5119. args = map[string]interface{}{
  5120. "bucketName": bucketName,
  5121. }
  5122. err = c.RemoveBucket(context.Background(), bucketName)
  5123. if err != nil {
  5124. logError(testName, function, args, startTime, "", "RemoveBucket failed", err)
  5125. return
  5126. }
  5127. err = c.RemoveBucket(context.Background(), bucketName)
  5128. if err == nil {
  5129. logError(testName, function, args, startTime, "", "RemoveBucket did not fail for invalid bucket name", err)
  5130. return
  5131. }
  5132. if err.Error() != "The specified bucket does not exist" {
  5133. logError(testName, function, args, startTime, "", "RemoveBucket failed", err)
  5134. return
  5135. }
  5136. os.Remove(fileName)
  5137. os.Remove(fileName + "-f")
  5138. successLogger(testName, functionAll, args, startTime).Info()
  5139. }
  5140. // Test for validating GetObject Reader* methods functioning when the
  5141. // object is modified in the object store.
  5142. func testGetObjectModified() {
  5143. // initialize logging params
  5144. startTime := time.Now()
  5145. testName := getFuncName()
  5146. function := "GetObject(bucketName, objectName)"
  5147. args := map[string]interface{}{}
  5148. // Instantiate new minio client object.
  5149. c, err := minio.New(os.Getenv(serverEndpoint),
  5150. &minio.Options{
  5151. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  5152. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  5153. })
  5154. if err != nil {
  5155. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  5156. return
  5157. }
  5158. // Enable tracing, write to stderr.
  5159. // c.TraceOn(os.Stderr)
  5160. // Set user agent.
  5161. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  5162. // Make a new bucket.
  5163. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  5164. args["bucketName"] = bucketName
  5165. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  5166. if err != nil {
  5167. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  5168. return
  5169. }
  5170. defer cleanupBucket(bucketName, c)
  5171. // Upload an object.
  5172. objectName := "myobject"
  5173. args["objectName"] = objectName
  5174. content := "helloworld"
  5175. _, err = c.PutObject(context.Background(), bucketName, objectName, strings.NewReader(content), int64(len(content)), minio.PutObjectOptions{ContentType: "application/text"})
  5176. if err != nil {
  5177. logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err)
  5178. return
  5179. }
  5180. defer c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{})
  5181. reader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  5182. if err != nil {
  5183. logError(testName, function, args, startTime, "", "Failed to GetObject "+objectName+", from bucket "+bucketName, err)
  5184. return
  5185. }
  5186. defer reader.Close()
  5187. // Read a few bytes of the object.
  5188. b := make([]byte, 5)
  5189. n, err := reader.ReadAt(b, 0)
  5190. if err != nil {
  5191. logError(testName, function, args, startTime, "", "Failed to read object "+objectName+", from bucket "+bucketName+" at an offset", err)
  5192. return
  5193. }
  5194. // Upload different contents to the same object while object is being read.
  5195. newContent := "goodbyeworld"
  5196. _, err = c.PutObject(context.Background(), bucketName, objectName, strings.NewReader(newContent), int64(len(newContent)), minio.PutObjectOptions{ContentType: "application/text"})
  5197. if err != nil {
  5198. logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err)
  5199. return
  5200. }
  5201. // Confirm that a Stat() call in between doesn't change the Object's cached etag.
  5202. _, err = reader.Stat()
  5203. expectedError := "At least one of the pre-conditions you specified did not hold"
  5204. if err.Error() != expectedError {
  5205. logError(testName, function, args, startTime, "", "Expected Stat to fail with error "+expectedError+", but received "+err.Error(), err)
  5206. return
  5207. }
  5208. // Read again only to find object contents have been modified since last read.
  5209. _, err = reader.ReadAt(b, int64(n))
  5210. if err.Error() != expectedError {
  5211. logError(testName, function, args, startTime, "", "Expected ReadAt to fail with error "+expectedError+", but received "+err.Error(), err)
  5212. return
  5213. }
  5214. successLogger(testName, function, args, startTime).Info()
  5215. }
  5216. // Test validates putObject to upload a file seeked at a given offset.
  5217. func testPutObjectUploadSeekedObject() {
  5218. // initialize logging params
  5219. startTime := time.Now()
  5220. testName := getFuncName()
  5221. function := "PutObject(bucketName, objectName, fileToUpload, contentType)"
  5222. args := map[string]interface{}{
  5223. "bucketName": "",
  5224. "objectName": "",
  5225. "fileToUpload": "",
  5226. "contentType": "binary/octet-stream",
  5227. }
  5228. // Instantiate new minio client object.
  5229. c, err := minio.New(os.Getenv(serverEndpoint),
  5230. &minio.Options{
  5231. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  5232. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  5233. })
  5234. if err != nil {
  5235. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  5236. return
  5237. }
  5238. // Enable tracing, write to stderr.
  5239. // c.TraceOn(os.Stderr)
  5240. // Set user agent.
  5241. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  5242. // Make a new bucket.
  5243. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  5244. args["bucketName"] = bucketName
  5245. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  5246. if err != nil {
  5247. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  5248. return
  5249. }
  5250. defer cleanupBucket(bucketName, c)
  5251. var tempfile *os.File
  5252. if fileName := getMintDataDirFilePath("datafile-100-kB"); fileName != "" {
  5253. tempfile, err = os.Open(fileName)
  5254. if err != nil {
  5255. logError(testName, function, args, startTime, "", "File open failed", err)
  5256. return
  5257. }
  5258. args["fileToUpload"] = fileName
  5259. } else {
  5260. tempfile, err = ioutil.TempFile("", "minio-go-upload-test-")
  5261. if err != nil {
  5262. logError(testName, function, args, startTime, "", "TempFile create failed", err)
  5263. return
  5264. }
  5265. args["fileToUpload"] = tempfile.Name()
  5266. // Generate 100kB data
  5267. if _, err = io.Copy(tempfile, getDataReader("datafile-100-kB")); err != nil {
  5268. logError(testName, function, args, startTime, "", "File copy failed", err)
  5269. return
  5270. }
  5271. defer os.Remove(tempfile.Name())
  5272. // Seek back to the beginning of the file.
  5273. tempfile.Seek(0, 0)
  5274. }
  5275. var length = 100 * humanize.KiByte
  5276. objectName := fmt.Sprintf("test-file-%v", rand.Uint32())
  5277. args["objectName"] = objectName
  5278. offset := length / 2
  5279. if _, err = tempfile.Seek(int64(offset), 0); err != nil {
  5280. logError(testName, function, args, startTime, "", "TempFile seek failed", err)
  5281. return
  5282. }
  5283. _, err = c.PutObject(context.Background(), bucketName, objectName, tempfile, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  5284. if err != nil {
  5285. logError(testName, function, args, startTime, "", "PutObject failed", err)
  5286. return
  5287. }
  5288. tempfile.Close()
  5289. obj, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  5290. if err != nil {
  5291. logError(testName, function, args, startTime, "", "GetObject failed", err)
  5292. return
  5293. }
  5294. defer obj.Close()
  5295. n, err := obj.Seek(int64(offset), 0)
  5296. if err != nil {
  5297. logError(testName, function, args, startTime, "", "Seek failed", err)
  5298. return
  5299. }
  5300. if n != int64(offset) {
  5301. logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(offset), n), err)
  5302. return
  5303. }
  5304. _, err = c.PutObject(context.Background(), bucketName, objectName+"getobject", obj, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  5305. if err != nil {
  5306. logError(testName, function, args, startTime, "", "PutObject failed", err)
  5307. return
  5308. }
  5309. st, err := c.StatObject(context.Background(), bucketName, objectName+"getobject", minio.StatObjectOptions{})
  5310. if err != nil {
  5311. logError(testName, function, args, startTime, "", "StatObject failed", err)
  5312. return
  5313. }
  5314. if st.Size != int64(length-offset) {
  5315. logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(length-offset), n), err)
  5316. return
  5317. }
  5318. successLogger(testName, function, args, startTime).Info()
  5319. }
  5320. // Tests bucket re-create errors.
  5321. func testMakeBucketErrorV2() {
  5322. // initialize logging params
  5323. startTime := time.Now()
  5324. testName := getFuncName()
  5325. function := "MakeBucket(bucketName, region)"
  5326. args := map[string]interface{}{
  5327. "bucketName": "",
  5328. "region": "eu-west-1",
  5329. }
  5330. // Seed random based on current time.
  5331. rand.Seed(time.Now().Unix())
  5332. // Instantiate new minio client object.
  5333. c, err := minio.New(os.Getenv(serverEndpoint),
  5334. &minio.Options{
  5335. Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  5336. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  5337. })
  5338. if err != nil {
  5339. logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
  5340. return
  5341. }
  5342. // Enable tracing, write to stderr.
  5343. // c.TraceOn(os.Stderr)
  5344. // Set user agent.
  5345. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  5346. // Generate a new random bucket name.
  5347. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  5348. region := "eu-west-1"
  5349. args["bucketName"] = bucketName
  5350. args["region"] = region
  5351. // Make a new bucket in 'eu-west-1'.
  5352. if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err != nil {
  5353. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  5354. return
  5355. }
  5356. defer cleanupBucket(bucketName, c)
  5357. if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err == nil {
  5358. logError(testName, function, args, startTime, "", "MakeBucket did not fail for existing bucket name", err)
  5359. return
  5360. }
  5361. // Verify valid error response from server.
  5362. if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" &&
  5363. minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" {
  5364. logError(testName, function, args, startTime, "", "Invalid error returned by server", err)
  5365. return
  5366. }
  5367. successLogger(testName, function, args, startTime).Info()
  5368. }
  5369. // Test get object reader to not throw error on being closed twice.
  5370. func testGetObjectClosedTwiceV2() {
  5371. // initialize logging params
  5372. startTime := time.Now()
  5373. testName := getFuncName()
  5374. function := "MakeBucket(bucketName, region)"
  5375. args := map[string]interface{}{
  5376. "bucketName": "",
  5377. "region": "eu-west-1",
  5378. }
  5379. // Seed random based on current time.
  5380. rand.Seed(time.Now().Unix())
  5381. // Instantiate new minio client object.
  5382. c, err := minio.New(os.Getenv(serverEndpoint),
  5383. &minio.Options{
  5384. Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  5385. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  5386. })
  5387. if err != nil {
  5388. logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
  5389. return
  5390. }
  5391. // Enable tracing, write to stderr.
  5392. // c.TraceOn(os.Stderr)
  5393. // Set user agent.
  5394. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  5395. // Generate a new random bucket name.
  5396. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  5397. args["bucketName"] = bucketName
  5398. // Make a new bucket.
  5399. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  5400. if err != nil {
  5401. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  5402. return
  5403. }
  5404. defer cleanupBucket(bucketName, c)
  5405. // Generate 33K of data.
  5406. bufSize := dataFileMap["datafile-33-kB"]
  5407. var reader = getDataReader("datafile-33-kB")
  5408. defer reader.Close()
  5409. // Save the data
  5410. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  5411. args["objectName"] = objectName
  5412. _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  5413. if err != nil {
  5414. logError(testName, function, args, startTime, "", "PutObject failed", err)
  5415. return
  5416. }
  5417. // Read the data back
  5418. r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  5419. if err != nil {
  5420. logError(testName, function, args, startTime, "", "GetObject failed", err)
  5421. return
  5422. }
  5423. st, err := r.Stat()
  5424. if err != nil {
  5425. logError(testName, function, args, startTime, "", "Stat failed", err)
  5426. return
  5427. }
  5428. if st.Size != int64(bufSize) {
  5429. logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(st.Size), err)
  5430. return
  5431. }
  5432. if err := r.Close(); err != nil {
  5433. logError(testName, function, args, startTime, "", "Stat failed", err)
  5434. return
  5435. }
  5436. if err := r.Close(); err == nil {
  5437. logError(testName, function, args, startTime, "", "Object is already closed, should return error", err)
  5438. return
  5439. }
  5440. successLogger(testName, function, args, startTime).Info()
  5441. }
  5442. // Tests FPutObject hidden contentType setting
  5443. func testFPutObjectV2() {
  5444. // initialize logging params
  5445. startTime := time.Now()
  5446. testName := getFuncName()
  5447. function := "FPutObject(bucketName, objectName, fileName, opts)"
  5448. args := map[string]interface{}{
  5449. "bucketName": "",
  5450. "objectName": "",
  5451. "fileName": "",
  5452. "opts": "",
  5453. }
  5454. // Seed random based on current time.
  5455. rand.Seed(time.Now().Unix())
  5456. // Instantiate new minio client object.
  5457. c, err := minio.New(os.Getenv(serverEndpoint),
  5458. &minio.Options{
  5459. Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  5460. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  5461. })
  5462. if err != nil {
  5463. logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
  5464. return
  5465. }
  5466. // Enable tracing, write to stderr.
  5467. // c.TraceOn(os.Stderr)
  5468. // Set user agent.
  5469. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  5470. // Generate a new random bucket name.
  5471. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  5472. args["bucketName"] = bucketName
  5473. // Make a new bucket.
  5474. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  5475. if err != nil {
  5476. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  5477. return
  5478. }
  5479. defer cleanupBucket(bucketName, c)
  5480. // Make a temp file with 11*1024*1024 bytes of data.
  5481. file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest")
  5482. if err != nil {
  5483. logError(testName, function, args, startTime, "", "TempFile creation failed", err)
  5484. return
  5485. }
  5486. r := bytes.NewReader(bytes.Repeat([]byte("b"), 11*1024*1024))
  5487. n, err := io.CopyN(file, r, 11*1024*1024)
  5488. if err != nil {
  5489. logError(testName, function, args, startTime, "", "Copy failed", err)
  5490. return
  5491. }
  5492. if n != int64(11*1024*1024) {
  5493. logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err)
  5494. return
  5495. }
  5496. // Close the file pro-actively for windows.
  5497. err = file.Close()
  5498. if err != nil {
  5499. logError(testName, function, args, startTime, "", "File close failed", err)
  5500. return
  5501. }
  5502. // Set base object name
  5503. objectName := bucketName + "FPutObject"
  5504. args["objectName"] = objectName
  5505. args["fileName"] = file.Name()
  5506. // Perform standard FPutObject with contentType provided (Expecting application/octet-stream)
  5507. _, err = c.FPutObject(context.Background(), bucketName, objectName+"-standard", file.Name(), minio.PutObjectOptions{ContentType: "application/octet-stream"})
  5508. if err != nil {
  5509. logError(testName, function, args, startTime, "", "FPutObject failed", err)
  5510. return
  5511. }
  5512. // Perform FPutObject with no contentType provided (Expecting application/octet-stream)
  5513. args["objectName"] = objectName + "-Octet"
  5514. args["contentType"] = ""
  5515. _, err = c.FPutObject(context.Background(), bucketName, objectName+"-Octet", file.Name(), minio.PutObjectOptions{})
  5516. if err != nil {
  5517. logError(testName, function, args, startTime, "", "FPutObject failed", err)
  5518. return
  5519. }
  5520. // Add extension to temp file name
  5521. fileName := file.Name()
  5522. err = os.Rename(fileName, fileName+".gtar")
  5523. if err != nil {
  5524. logError(testName, function, args, startTime, "", "Rename failed", err)
  5525. return
  5526. }
  5527. // Perform FPutObject with no contentType provided (Expecting application/x-gtar)
  5528. args["objectName"] = objectName + "-Octet"
  5529. args["contentType"] = ""
  5530. args["fileName"] = fileName + ".gtar"
  5531. _, err = c.FPutObject(context.Background(), bucketName, objectName+"-GTar", fileName+".gtar", minio.PutObjectOptions{})
  5532. if err != nil {
  5533. logError(testName, function, args, startTime, "", "FPutObject failed", err)
  5534. return
  5535. }
  5536. // Check headers and sizes
  5537. rStandard, err := c.StatObject(context.Background(), bucketName, objectName+"-standard", minio.StatObjectOptions{})
  5538. if err != nil {
  5539. logError(testName, function, args, startTime, "", "StatObject failed", err)
  5540. return
  5541. }
  5542. if rStandard.Size != 11*1024*1024 {
  5543. logError(testName, function, args, startTime, "", "Unexpected size", nil)
  5544. return
  5545. }
  5546. if rStandard.ContentType != "application/octet-stream" {
  5547. logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rStandard.ContentType, err)
  5548. return
  5549. }
  5550. rOctet, err := c.StatObject(context.Background(), bucketName, objectName+"-Octet", minio.StatObjectOptions{})
  5551. if err != nil {
  5552. logError(testName, function, args, startTime, "", "StatObject failed", err)
  5553. return
  5554. }
  5555. if rOctet.ContentType != "application/octet-stream" {
  5556. logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rOctet.ContentType, err)
  5557. return
  5558. }
  5559. if rOctet.Size != 11*1024*1024 {
  5560. logError(testName, function, args, startTime, "", "Unexpected size", nil)
  5561. return
  5562. }
  5563. rGTar, err := c.StatObject(context.Background(), bucketName, objectName+"-GTar", minio.StatObjectOptions{})
  5564. if err != nil {
  5565. logError(testName, function, args, startTime, "", "StatObject failed", err)
  5566. return
  5567. }
  5568. if rGTar.Size != 11*1024*1024 {
  5569. logError(testName, function, args, startTime, "", "Unexpected size", nil)
  5570. return
  5571. }
  5572. if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" {
  5573. logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/x-gtar , got "+rGTar.ContentType, err)
  5574. return
  5575. }
  5576. os.Remove(fileName + ".gtar")
  5577. successLogger(testName, function, args, startTime).Info()
  5578. }
  5579. // Tests various bucket supported formats.
  5580. func testMakeBucketRegionsV2() {
  5581. // initialize logging params
  5582. startTime := time.Now()
  5583. testName := getFuncName()
  5584. function := "MakeBucket(bucketName, region)"
  5585. args := map[string]interface{}{
  5586. "bucketName": "",
  5587. "region": "eu-west-1",
  5588. }
  5589. // Seed random based on current time.
  5590. rand.Seed(time.Now().Unix())
  5591. // Instantiate new minio client object.
  5592. c, err := minio.New(os.Getenv(serverEndpoint),
  5593. &minio.Options{
  5594. Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  5595. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  5596. })
  5597. if err != nil {
  5598. logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
  5599. return
  5600. }
  5601. // Enable tracing, write to stderr.
  5602. // c.TraceOn(os.Stderr)
  5603. // Set user agent.
  5604. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  5605. // Generate a new random bucket name.
  5606. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  5607. args["bucketName"] = bucketName
  5608. // Make a new bucket in 'eu-central-1'.
  5609. if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "eu-west-1"}); err != nil {
  5610. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  5611. return
  5612. }
  5613. if err = cleanupBucket(bucketName, c); err != nil {
  5614. logError(testName, function, args, startTime, "", "CleanupBucket failed while removing bucket recursively", err)
  5615. return
  5616. }
  5617. // Make a new bucket with '.' in its name, in 'us-west-2'. This
  5618. // request is internally staged into a path style instead of
  5619. // virtual host style.
  5620. if err = c.MakeBucket(context.Background(), bucketName+".withperiod", minio.MakeBucketOptions{Region: "us-west-2"}); err != nil {
  5621. args["bucketName"] = bucketName + ".withperiod"
  5622. args["region"] = "us-west-2"
  5623. logError(testName, function, args, startTime, "", "MakeBucket test with a bucket name with period, '.', failed", err)
  5624. return
  5625. }
  5626. // Delete all objects and buckets
  5627. if err = cleanupBucket(bucketName+".withperiod", c); err != nil {
  5628. logError(testName, function, args, startTime, "", "CleanupBucket failed while removing bucket recursively", err)
  5629. return
  5630. }
  5631. successLogger(testName, function, args, startTime).Info()
  5632. }
  5633. // Tests get object ReaderSeeker interface methods.
  5634. func testGetObjectReadSeekFunctionalV2() {
  5635. // initialize logging params
  5636. startTime := time.Now()
  5637. testName := getFuncName()
  5638. function := "GetObject(bucketName, objectName)"
  5639. args := map[string]interface{}{}
  5640. // Seed random based on current time.
  5641. rand.Seed(time.Now().Unix())
  5642. // Instantiate new minio client object.
  5643. c, err := minio.New(os.Getenv(serverEndpoint),
  5644. &minio.Options{
  5645. Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  5646. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  5647. })
  5648. if err != nil {
  5649. logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
  5650. return
  5651. }
  5652. // Enable tracing, write to stderr.
  5653. // c.TraceOn(os.Stderr)
  5654. // Set user agent.
  5655. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  5656. // Generate a new random bucket name.
  5657. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  5658. args["bucketName"] = bucketName
  5659. // Make a new bucket.
  5660. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  5661. if err != nil {
  5662. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  5663. return
  5664. }
  5665. defer cleanupBucket(bucketName, c)
  5666. // Generate 33K of data.
  5667. bufSize := dataFileMap["datafile-33-kB"]
  5668. var reader = getDataReader("datafile-33-kB")
  5669. defer reader.Close()
  5670. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  5671. args["objectName"] = objectName
  5672. buf, err := ioutil.ReadAll(reader)
  5673. if err != nil {
  5674. logError(testName, function, args, startTime, "", "ReadAll failed", err)
  5675. return
  5676. }
  5677. // Save the data.
  5678. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  5679. if err != nil {
  5680. logError(testName, function, args, startTime, "", "PutObject failed", err)
  5681. return
  5682. }
  5683. // Read the data back
  5684. r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  5685. if err != nil {
  5686. logError(testName, function, args, startTime, "", "GetObject failed", err)
  5687. return
  5688. }
  5689. defer r.Close()
  5690. st, err := r.Stat()
  5691. if err != nil {
  5692. logError(testName, function, args, startTime, "", "Stat failed", err)
  5693. return
  5694. }
  5695. if st.Size != int64(bufSize) {
  5696. logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err)
  5697. return
  5698. }
  5699. offset := int64(2048)
  5700. n, err := r.Seek(offset, 0)
  5701. if err != nil {
  5702. logError(testName, function, args, startTime, "", "Seek failed", err)
  5703. return
  5704. }
  5705. if n != offset {
  5706. logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err)
  5707. return
  5708. }
  5709. n, err = r.Seek(0, 1)
  5710. if err != nil {
  5711. logError(testName, function, args, startTime, "", "Seek failed", err)
  5712. return
  5713. }
  5714. if n != offset {
  5715. logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err)
  5716. return
  5717. }
  5718. _, err = r.Seek(offset, 2)
  5719. if err == nil {
  5720. logError(testName, function, args, startTime, "", "Seek on positive offset for whence '2' should error out", err)
  5721. return
  5722. }
  5723. n, err = r.Seek(-offset, 2)
  5724. if err != nil {
  5725. logError(testName, function, args, startTime, "", "Seek failed", err)
  5726. return
  5727. }
  5728. if n != st.Size-offset {
  5729. logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(st.Size-offset)+" got "+string(n), err)
  5730. return
  5731. }
  5732. var buffer1 bytes.Buffer
  5733. if _, err = io.CopyN(&buffer1, r, st.Size); err != nil {
  5734. if err != io.EOF {
  5735. logError(testName, function, args, startTime, "", "Copy failed", err)
  5736. return
  5737. }
  5738. }
  5739. if !bytes.Equal(buf[len(buf)-int(offset):], buffer1.Bytes()) {
  5740. logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err)
  5741. return
  5742. }
  5743. // Seek again and read again.
  5744. n, err = r.Seek(offset-1, 0)
  5745. if err != nil {
  5746. logError(testName, function, args, startTime, "", "Seek failed", err)
  5747. return
  5748. }
  5749. if n != (offset - 1) {
  5750. logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset-1)+" got "+string(n), err)
  5751. return
  5752. }
  5753. var buffer2 bytes.Buffer
  5754. if _, err = io.CopyN(&buffer2, r, st.Size); err != nil {
  5755. if err != io.EOF {
  5756. logError(testName, function, args, startTime, "", "Copy failed", err)
  5757. return
  5758. }
  5759. }
  5760. // Verify now lesser bytes.
  5761. if !bytes.Equal(buf[2047:], buffer2.Bytes()) {
  5762. logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err)
  5763. return
  5764. }
  5765. successLogger(testName, function, args, startTime).Info()
  5766. }
  5767. // Tests get object ReaderAt interface methods.
  5768. func testGetObjectReadAtFunctionalV2() {
  5769. // initialize logging params
  5770. startTime := time.Now()
  5771. testName := getFuncName()
  5772. function := "GetObject(bucketName, objectName)"
  5773. args := map[string]interface{}{}
  5774. // Seed random based on current time.
  5775. rand.Seed(time.Now().Unix())
  5776. // Instantiate new minio client object.
  5777. c, err := minio.New(os.Getenv(serverEndpoint),
  5778. &minio.Options{
  5779. Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  5780. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  5781. })
  5782. if err != nil {
  5783. logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
  5784. return
  5785. }
  5786. // Enable tracing, write to stderr.
  5787. // c.TraceOn(os.Stderr)
  5788. // Set user agent.
  5789. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  5790. // Generate a new random bucket name.
  5791. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  5792. args["bucketName"] = bucketName
  5793. // Make a new bucket.
  5794. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  5795. if err != nil {
  5796. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  5797. return
  5798. }
  5799. defer cleanupBucket(bucketName, c)
  5800. // Generate 33K of data.
  5801. bufSize := dataFileMap["datafile-33-kB"]
  5802. var reader = getDataReader("datafile-33-kB")
  5803. defer reader.Close()
  5804. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  5805. args["objectName"] = objectName
  5806. buf, err := ioutil.ReadAll(reader)
  5807. if err != nil {
  5808. logError(testName, function, args, startTime, "", "ReadAll failed", err)
  5809. return
  5810. }
  5811. // Save the data
  5812. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  5813. if err != nil {
  5814. logError(testName, function, args, startTime, "", "PutObject failed", err)
  5815. return
  5816. }
  5817. // Read the data back
  5818. r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  5819. if err != nil {
  5820. logError(testName, function, args, startTime, "", "GetObject failed", err)
  5821. return
  5822. }
  5823. defer r.Close()
  5824. st, err := r.Stat()
  5825. if err != nil {
  5826. logError(testName, function, args, startTime, "", "Stat failed", err)
  5827. return
  5828. }
  5829. if st.Size != int64(bufSize) {
  5830. logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(st.Size), err)
  5831. return
  5832. }
  5833. offset := int64(2048)
  5834. // Read directly
  5835. buf2 := make([]byte, 512)
  5836. buf3 := make([]byte, 512)
  5837. buf4 := make([]byte, 512)
  5838. m, err := r.ReadAt(buf2, offset)
  5839. if err != nil {
  5840. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  5841. return
  5842. }
  5843. if m != len(buf2) {
  5844. logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+" got "+string(m), err)
  5845. return
  5846. }
  5847. if !bytes.Equal(buf2, buf[offset:offset+512]) {
  5848. logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
  5849. return
  5850. }
  5851. offset += 512
  5852. m, err = r.ReadAt(buf3, offset)
  5853. if err != nil {
  5854. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  5855. return
  5856. }
  5857. if m != len(buf3) {
  5858. logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+" got "+string(m), err)
  5859. return
  5860. }
  5861. if !bytes.Equal(buf3, buf[offset:offset+512]) {
  5862. logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
  5863. return
  5864. }
  5865. offset += 512
  5866. m, err = r.ReadAt(buf4, offset)
  5867. if err != nil {
  5868. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  5869. return
  5870. }
  5871. if m != len(buf4) {
  5872. logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+" got "+string(m), err)
  5873. return
  5874. }
  5875. if !bytes.Equal(buf4, buf[offset:offset+512]) {
  5876. logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
  5877. return
  5878. }
  5879. buf5 := make([]byte, bufSize)
  5880. // Read the whole object.
  5881. m, err = r.ReadAt(buf5, 0)
  5882. if err != nil {
  5883. if err != io.EOF {
  5884. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  5885. return
  5886. }
  5887. }
  5888. if m != len(buf5) {
  5889. logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+" got "+string(m), err)
  5890. return
  5891. }
  5892. if !bytes.Equal(buf, buf5) {
  5893. logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err)
  5894. return
  5895. }
  5896. buf6 := make([]byte, bufSize+1)
  5897. // Read the whole object and beyond.
  5898. _, err = r.ReadAt(buf6, 0)
  5899. if err != nil {
  5900. if err != io.EOF {
  5901. logError(testName, function, args, startTime, "", "ReadAt failed", err)
  5902. return
  5903. }
  5904. }
  5905. successLogger(testName, function, args, startTime).Info()
  5906. }
  5907. // Tests copy object
  5908. func testCopyObjectV2() {
  5909. // initialize logging params
  5910. startTime := time.Now()
  5911. testName := getFuncName()
  5912. function := "CopyObject(destination, source)"
  5913. args := map[string]interface{}{}
  5914. // Seed random based on current time.
  5915. rand.Seed(time.Now().Unix())
  5916. // Instantiate new minio client object
  5917. c, err := minio.New(os.Getenv(serverEndpoint),
  5918. &minio.Options{
  5919. Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  5920. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  5921. })
  5922. if err != nil {
  5923. logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
  5924. return
  5925. }
  5926. // Enable tracing, write to stderr.
  5927. // c.TraceOn(os.Stderr)
  5928. // Set user agent.
  5929. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  5930. // Generate a new random bucket name.
  5931. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  5932. // Make a new bucket in 'us-east-1' (source bucket).
  5933. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  5934. if err != nil {
  5935. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  5936. return
  5937. }
  5938. defer cleanupBucket(bucketName, c)
  5939. // Make a new bucket in 'us-east-1' (destination bucket).
  5940. err = c.MakeBucket(context.Background(), bucketName+"-copy", minio.MakeBucketOptions{Region: "us-east-1"})
  5941. if err != nil {
  5942. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  5943. return
  5944. }
  5945. defer cleanupBucket(bucketName+"-copy", c)
  5946. // Generate 33K of data.
  5947. bufSize := dataFileMap["datafile-33-kB"]
  5948. var reader = getDataReader("datafile-33-kB")
  5949. defer reader.Close()
  5950. // Save the data
  5951. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  5952. _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  5953. if err != nil {
  5954. logError(testName, function, args, startTime, "", "PutObject failed", err)
  5955. return
  5956. }
  5957. r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  5958. if err != nil {
  5959. logError(testName, function, args, startTime, "", "GetObject failed", err)
  5960. return
  5961. }
  5962. // Check the various fields of source object against destination object.
  5963. objInfo, err := r.Stat()
  5964. if err != nil {
  5965. logError(testName, function, args, startTime, "", "Stat failed", err)
  5966. return
  5967. }
  5968. r.Close()
  5969. // Copy Source
  5970. src := minio.CopySrcOptions{
  5971. Bucket: bucketName,
  5972. Object: objectName,
  5973. MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC),
  5974. MatchETag: objInfo.ETag,
  5975. }
  5976. args["source"] = src
  5977. // Set copy conditions.
  5978. dst := minio.CopyDestOptions{
  5979. Bucket: bucketName + "-copy",
  5980. Object: objectName + "-copy",
  5981. }
  5982. args["destination"] = dst
  5983. // Perform the Copy
  5984. _, err = c.CopyObject(context.Background(), dst, src)
  5985. if err != nil {
  5986. logError(testName, function, args, startTime, "", "CopyObject failed", err)
  5987. return
  5988. }
  5989. // Source object
  5990. r, err = c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  5991. if err != nil {
  5992. logError(testName, function, args, startTime, "", "GetObject failed", err)
  5993. return
  5994. }
  5995. // Destination object
  5996. readerCopy, err := c.GetObject(context.Background(), bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{})
  5997. if err != nil {
  5998. logError(testName, function, args, startTime, "", "GetObject failed", err)
  5999. return
  6000. }
  6001. // Check the various fields of source object against destination object.
  6002. objInfo, err = r.Stat()
  6003. if err != nil {
  6004. logError(testName, function, args, startTime, "", "Stat failed", err)
  6005. return
  6006. }
  6007. objInfoCopy, err := readerCopy.Stat()
  6008. if err != nil {
  6009. logError(testName, function, args, startTime, "", "Stat failed", err)
  6010. return
  6011. }
  6012. if objInfo.Size != objInfoCopy.Size {
  6013. logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(objInfoCopy.Size)+" got "+string(objInfo.Size), err)
  6014. return
  6015. }
  6016. // Close all the readers.
  6017. r.Close()
  6018. readerCopy.Close()
  6019. // CopyObject again but with wrong conditions
  6020. src = minio.CopySrcOptions{
  6021. Bucket: bucketName,
  6022. Object: objectName,
  6023. MatchUnmodifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC),
  6024. NoMatchETag: objInfo.ETag,
  6025. }
  6026. // Perform the Copy which should fail
  6027. _, err = c.CopyObject(context.Background(), dst, src)
  6028. if err == nil {
  6029. logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err)
  6030. return
  6031. }
  6032. successLogger(testName, function, args, startTime).Info()
  6033. }
  6034. func testComposeObjectErrorCasesWrapper(c *minio.Client) {
  6035. // initialize logging params
  6036. startTime := time.Now()
  6037. testName := getFuncName()
  6038. function := "ComposeObject(destination, sourceList)"
  6039. args := map[string]interface{}{}
  6040. // Generate a new random bucket name.
  6041. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  6042. // Make a new bucket in 'us-east-1' (source bucket).
  6043. err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  6044. if err != nil {
  6045. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  6046. return
  6047. }
  6048. defer cleanupBucket(bucketName, c)
  6049. // Test that more than 10K source objects cannot be
  6050. // concatenated.
  6051. srcArr := [10001]minio.CopySrcOptions{}
  6052. srcSlice := srcArr[:]
  6053. dst := minio.CopyDestOptions{
  6054. Bucket: bucketName,
  6055. Object: "object",
  6056. }
  6057. args["destination"] = dst
  6058. // Just explain about srcArr in args["sourceList"]
  6059. // to stop having 10,001 null headers logged
  6060. args["sourceList"] = "source array of 10,001 elements"
  6061. if _, err := c.ComposeObject(context.Background(), dst, srcSlice...); err == nil {
  6062. logError(testName, function, args, startTime, "", "Expected error in ComposeObject", err)
  6063. return
  6064. } else if err.Error() != "There must be as least one and up to 10000 source objects." {
  6065. logError(testName, function, args, startTime, "", "Got unexpected error", err)
  6066. return
  6067. }
  6068. // Create a source with invalid offset spec and check that
  6069. // error is returned:
  6070. // 1. Create the source object.
  6071. const badSrcSize = 5 * 1024 * 1024
  6072. buf := bytes.Repeat([]byte("1"), badSrcSize)
  6073. _, err = c.PutObject(context.Background(), bucketName, "badObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{})
  6074. if err != nil {
  6075. logError(testName, function, args, startTime, "", "PutObject failed", err)
  6076. return
  6077. }
  6078. // 2. Set invalid range spec on the object (going beyond
  6079. // object size)
  6080. badSrc := minio.CopySrcOptions{
  6081. Bucket: bucketName,
  6082. Object: "badObject",
  6083. MatchRange: true,
  6084. Start: 1,
  6085. End: badSrcSize,
  6086. }
  6087. // 3. ComposeObject call should fail.
  6088. if _, err := c.ComposeObject(context.Background(), dst, badSrc); err == nil {
  6089. logError(testName, function, args, startTime, "", "ComposeObject expected to fail", err)
  6090. return
  6091. } else if !strings.Contains(err.Error(), "has invalid segment-to-copy") {
  6092. logError(testName, function, args, startTime, "", "Got invalid error", err)
  6093. return
  6094. }
  6095. successLogger(testName, function, args, startTime).Info()
  6096. }
  6097. // Test expected error cases
  6098. func testComposeObjectErrorCasesV2() {
  6099. // initialize logging params
  6100. startTime := time.Now()
  6101. testName := getFuncName()
  6102. function := "ComposeObject(destination, sourceList)"
  6103. args := map[string]interface{}{}
  6104. // Instantiate new minio client object
  6105. c, err := minio.New(os.Getenv(serverEndpoint),
  6106. &minio.Options{
  6107. Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  6108. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  6109. })
  6110. if err != nil {
  6111. logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
  6112. return
  6113. }
  6114. testComposeObjectErrorCasesWrapper(c)
  6115. }
  6116. func testComposeMultipleSources(c *minio.Client) {
  6117. // initialize logging params
  6118. startTime := time.Now()
  6119. testName := getFuncName()
  6120. function := "ComposeObject(destination, sourceList)"
  6121. args := map[string]interface{}{
  6122. "destination": "",
  6123. "sourceList": "",
  6124. }
  6125. // Generate a new random bucket name.
  6126. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  6127. // Make a new bucket in 'us-east-1' (source bucket).
  6128. err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  6129. if err != nil {
  6130. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  6131. return
  6132. }
  6133. defer cleanupBucket(bucketName, c)
  6134. // Upload a small source object
  6135. const srcSize = 1024 * 1024 * 5
  6136. buf := bytes.Repeat([]byte("1"), srcSize)
  6137. _, err = c.PutObject(context.Background(), bucketName, "srcObject", bytes.NewReader(buf), int64(srcSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  6138. if err != nil {
  6139. logError(testName, function, args, startTime, "", "PutObject failed", err)
  6140. return
  6141. }
  6142. // We will append 10 copies of the object.
  6143. srcs := []minio.CopySrcOptions{}
  6144. for i := 0; i < 10; i++ {
  6145. srcs = append(srcs, minio.CopySrcOptions{
  6146. Bucket: bucketName,
  6147. Object: "srcObject",
  6148. })
  6149. }
  6150. // make the last part very small
  6151. srcs[9].MatchRange = true
  6152. args["sourceList"] = srcs
  6153. dst := minio.CopyDestOptions{
  6154. Bucket: bucketName,
  6155. Object: "dstObject",
  6156. }
  6157. args["destination"] = dst
  6158. ui, err := c.ComposeObject(context.Background(), dst, srcs...)
  6159. if err != nil {
  6160. logError(testName, function, args, startTime, "", "ComposeObject failed", err)
  6161. return
  6162. }
  6163. if ui.Size != 9*srcSize+1 {
  6164. logError(testName, function, args, startTime, "", "ComposeObject returned unexpected size", err)
  6165. return
  6166. }
  6167. objProps, err := c.StatObject(context.Background(), bucketName, "dstObject", minio.StatObjectOptions{})
  6168. if err != nil {
  6169. logError(testName, function, args, startTime, "", "StatObject failed", err)
  6170. return
  6171. }
  6172. if objProps.Size != 9*srcSize+1 {
  6173. logError(testName, function, args, startTime, "", "Size mismatched! Expected "+string(10000*srcSize)+" got "+string(objProps.Size), err)
  6174. return
  6175. }
  6176. successLogger(testName, function, args, startTime).Info()
  6177. }
  6178. // Test concatenating multiple 10K objects V2
  6179. func testCompose10KSourcesV2() {
  6180. // initialize logging params
  6181. startTime := time.Now()
  6182. testName := getFuncName()
  6183. function := "ComposeObject(destination, sourceList)"
  6184. args := map[string]interface{}{}
  6185. // Instantiate new minio client object
  6186. c, err := minio.New(os.Getenv(serverEndpoint),
  6187. &minio.Options{
  6188. Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  6189. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  6190. })
  6191. if err != nil {
  6192. logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
  6193. return
  6194. }
  6195. testComposeMultipleSources(c)
  6196. }
  6197. func testEncryptedEmptyObject() {
  6198. // initialize logging params
  6199. startTime := time.Now()
  6200. testName := getFuncName()
  6201. function := "PutObject(bucketName, objectName, reader, objectSize, opts)"
  6202. args := map[string]interface{}{}
  6203. // Instantiate new minio client object
  6204. c, err := minio.New(os.Getenv(serverEndpoint),
  6205. &minio.Options{
  6206. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  6207. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  6208. })
  6209. if err != nil {
  6210. logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
  6211. return
  6212. }
  6213. // Generate a new random bucket name.
  6214. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  6215. args["bucketName"] = bucketName
  6216. // Make a new bucket in 'us-east-1' (source bucket).
  6217. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  6218. if err != nil {
  6219. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  6220. return
  6221. }
  6222. defer cleanupBucket(bucketName, c)
  6223. sse := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"object"))
  6224. // 1. create an sse-c encrypted object to copy by uploading
  6225. const srcSize = 0
  6226. var buf []byte // Empty buffer
  6227. args["objectName"] = "object"
  6228. _, err = c.PutObject(context.Background(), bucketName, "object", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ServerSideEncryption: sse})
  6229. if err != nil {
  6230. logError(testName, function, args, startTime, "", "PutObject call failed", err)
  6231. return
  6232. }
  6233. // 2. Test CopyObject for an empty object
  6234. src := minio.CopySrcOptions{
  6235. Bucket: bucketName,
  6236. Object: "object",
  6237. Encryption: sse,
  6238. }
  6239. dst := minio.CopyDestOptions{
  6240. Bucket: bucketName,
  6241. Object: "new-object",
  6242. Encryption: sse,
  6243. }
  6244. if _, err = c.CopyObject(context.Background(), dst, src); err != nil {
  6245. function = "CopyObject(dst, src)"
  6246. logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject failed", err)
  6247. return
  6248. }
  6249. // 3. Test Key rotation
  6250. newSSE := encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"new-object"))
  6251. src = minio.CopySrcOptions{
  6252. Bucket: bucketName,
  6253. Object: "new-object",
  6254. Encryption: sse,
  6255. }
  6256. dst = minio.CopyDestOptions{
  6257. Bucket: bucketName,
  6258. Object: "new-object",
  6259. Encryption: newSSE,
  6260. }
  6261. if _, err = c.CopyObject(context.Background(), dst, src); err != nil {
  6262. function = "CopyObject(dst, src)"
  6263. logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject with key rotation failed", err)
  6264. return
  6265. }
  6266. // 4. Download the object.
  6267. reader, err := c.GetObject(context.Background(), bucketName, "new-object", minio.GetObjectOptions{ServerSideEncryption: newSSE})
  6268. if err != nil {
  6269. logError(testName, function, args, startTime, "", "GetObject failed", err)
  6270. return
  6271. }
  6272. defer reader.Close()
  6273. decBytes, err := ioutil.ReadAll(reader)
  6274. if err != nil {
  6275. logError(testName, function, map[string]interface{}{}, startTime, "", "ReadAll failed", err)
  6276. return
  6277. }
  6278. if !bytes.Equal(decBytes, buf) {
  6279. logError(testName, function, map[string]interface{}{}, startTime, "", "Downloaded object doesn't match the empty encrypted object", err)
  6280. return
  6281. }
  6282. delete(args, "objectName")
  6283. successLogger(testName, function, args, startTime).Info()
  6284. }
  6285. func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, sseDst encrypt.ServerSide) {
  6286. // initialize logging params
  6287. startTime := time.Now()
  6288. testName := getFuncNameLoc(2)
  6289. function := "CopyObject(destination, source)"
  6290. args := map[string]interface{}{}
  6291. var srcEncryption, dstEncryption encrypt.ServerSide
  6292. // Make a new bucket in 'us-east-1' (source bucket).
  6293. err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  6294. if err != nil {
  6295. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  6296. return
  6297. }
  6298. defer cleanupBucket(bucketName, c)
  6299. // 1. create an sse-c encrypted object to copy by uploading
  6300. const srcSize = 1024 * 1024
  6301. buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB
  6302. _, err = c.PutObject(context.Background(), bucketName, "srcObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{
  6303. ServerSideEncryption: sseSrc,
  6304. })
  6305. if err != nil {
  6306. logError(testName, function, args, startTime, "", "PutObject call failed", err)
  6307. return
  6308. }
  6309. if sseSrc != nil && sseSrc.Type() != encrypt.S3 {
  6310. srcEncryption = sseSrc
  6311. }
  6312. // 2. copy object and change encryption key
  6313. src := minio.CopySrcOptions{
  6314. Bucket: bucketName,
  6315. Object: "srcObject",
  6316. Encryption: srcEncryption,
  6317. }
  6318. args["source"] = src
  6319. dst := minio.CopyDestOptions{
  6320. Bucket: bucketName,
  6321. Object: "dstObject",
  6322. Encryption: sseDst,
  6323. }
  6324. args["destination"] = dst
  6325. _, err = c.CopyObject(context.Background(), dst, src)
  6326. if err != nil {
  6327. logError(testName, function, args, startTime, "", "CopyObject failed", err)
  6328. return
  6329. }
  6330. if sseDst != nil && sseDst.Type() != encrypt.S3 {
  6331. dstEncryption = sseDst
  6332. }
  6333. // 3. get copied object and check if content is equal
  6334. coreClient := minio.Core{c}
  6335. reader, _, _, err := coreClient.GetObject(context.Background(), bucketName, "dstObject", minio.GetObjectOptions{ServerSideEncryption: dstEncryption})
  6336. if err != nil {
  6337. logError(testName, function, args, startTime, "", "GetObject failed", err)
  6338. return
  6339. }
  6340. decBytes, err := ioutil.ReadAll(reader)
  6341. if err != nil {
  6342. logError(testName, function, args, startTime, "", "ReadAll failed", err)
  6343. return
  6344. }
  6345. if !bytes.Equal(decBytes, buf) {
  6346. logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err)
  6347. return
  6348. }
  6349. reader.Close()
  6350. // Test key rotation for source object in-place.
  6351. var newSSE encrypt.ServerSide
  6352. if sseSrc != nil && sseSrc.Type() == encrypt.SSEC {
  6353. newSSE = encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"srcObject")) // replace key
  6354. }
  6355. if sseSrc != nil && sseSrc.Type() == encrypt.S3 {
  6356. newSSE = encrypt.NewSSE()
  6357. }
  6358. if newSSE != nil {
  6359. dst = minio.CopyDestOptions{
  6360. Bucket: bucketName,
  6361. Object: "srcObject",
  6362. Encryption: newSSE,
  6363. }
  6364. args["destination"] = dst
  6365. _, err = c.CopyObject(context.Background(), dst, src)
  6366. if err != nil {
  6367. logError(testName, function, args, startTime, "", "CopyObject failed", err)
  6368. return
  6369. }
  6370. // Get copied object and check if content is equal
  6371. reader, _, _, err = coreClient.GetObject(context.Background(), bucketName, "srcObject", minio.GetObjectOptions{ServerSideEncryption: newSSE})
  6372. if err != nil {
  6373. logError(testName, function, args, startTime, "", "GetObject failed", err)
  6374. return
  6375. }
  6376. decBytes, err = ioutil.ReadAll(reader)
  6377. if err != nil {
  6378. logError(testName, function, args, startTime, "", "ReadAll failed", err)
  6379. return
  6380. }
  6381. if !bytes.Equal(decBytes, buf) {
  6382. logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err)
  6383. return
  6384. }
  6385. reader.Close()
  6386. // Test in-place decryption.
  6387. dst = minio.CopyDestOptions{
  6388. Bucket: bucketName,
  6389. Object: "srcObject",
  6390. }
  6391. args["destination"] = dst
  6392. src = minio.CopySrcOptions{
  6393. Bucket: bucketName,
  6394. Object: "srcObject",
  6395. Encryption: newSSE,
  6396. }
  6397. args["source"] = src
  6398. _, err = c.CopyObject(context.Background(), dst, src)
  6399. if err != nil {
  6400. logError(testName, function, args, startTime, "", "CopyObject Key rotation failed", err)
  6401. return
  6402. }
  6403. }
  6404. // Get copied decrypted object and check if content is equal
  6405. reader, _, _, err = coreClient.GetObject(context.Background(), bucketName, "srcObject", minio.GetObjectOptions{})
  6406. if err != nil {
  6407. logError(testName, function, args, startTime, "", "GetObject failed", err)
  6408. return
  6409. }
  6410. defer reader.Close()
  6411. decBytes, err = ioutil.ReadAll(reader)
  6412. if err != nil {
  6413. logError(testName, function, args, startTime, "", "ReadAll failed", err)
  6414. return
  6415. }
  6416. if !bytes.Equal(decBytes, buf) {
  6417. logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err)
  6418. return
  6419. }
  6420. successLogger(testName, function, args, startTime).Info()
  6421. }
  6422. // Test encrypted copy object
  6423. func testUnencryptedToSSECCopyObject() {
  6424. // initialize logging params
  6425. startTime := time.Now()
  6426. testName := getFuncName()
  6427. function := "CopyObject(destination, source)"
  6428. args := map[string]interface{}{}
  6429. // Instantiate new minio client object
  6430. c, err := minio.New(os.Getenv(serverEndpoint),
  6431. &minio.Options{
  6432. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  6433. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  6434. })
  6435. if err != nil {
  6436. logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
  6437. return
  6438. }
  6439. // Generate a new random bucket name.
  6440. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  6441. sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject"))
  6442. // c.TraceOn(os.Stderr)
  6443. testEncryptedCopyObjectWrapper(c, bucketName, nil, sseDst)
  6444. }
  6445. // Test encrypted copy object
  6446. func testUnencryptedToSSES3CopyObject() {
  6447. // initialize logging params
  6448. startTime := time.Now()
  6449. testName := getFuncName()
  6450. function := "CopyObject(destination, source)"
  6451. args := map[string]interface{}{}
  6452. // Instantiate new minio client object
  6453. c, err := minio.New(os.Getenv(serverEndpoint),
  6454. &minio.Options{
  6455. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  6456. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  6457. })
  6458. if err != nil {
  6459. logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
  6460. return
  6461. }
  6462. // Generate a new random bucket name.
  6463. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  6464. var sseSrc encrypt.ServerSide
  6465. sseDst := encrypt.NewSSE()
  6466. // c.TraceOn(os.Stderr)
  6467. testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
  6468. }
  6469. // Test encrypted copy object
  6470. func testUnencryptedToUnencryptedCopyObject() {
  6471. // initialize logging params
  6472. startTime := time.Now()
  6473. testName := getFuncName()
  6474. function := "CopyObject(destination, source)"
  6475. args := map[string]interface{}{}
  6476. // Instantiate new minio client object
  6477. c, err := minio.New(os.Getenv(serverEndpoint),
  6478. &minio.Options{
  6479. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  6480. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  6481. })
  6482. if err != nil {
  6483. logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
  6484. return
  6485. }
  6486. // Generate a new random bucket name.
  6487. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  6488. var sseSrc, sseDst encrypt.ServerSide
  6489. // c.TraceOn(os.Stderr)
  6490. testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
  6491. }
  6492. // Test encrypted copy object
  6493. func testEncryptedSSECToSSECCopyObject() {
  6494. // initialize logging params
  6495. startTime := time.Now()
  6496. testName := getFuncName()
  6497. function := "CopyObject(destination, source)"
  6498. args := map[string]interface{}{}
  6499. // Instantiate new minio client object
  6500. c, err := minio.New(os.Getenv(serverEndpoint),
  6501. &minio.Options{
  6502. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  6503. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  6504. })
  6505. if err != nil {
  6506. logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
  6507. return
  6508. }
  6509. // Generate a new random bucket name.
  6510. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  6511. sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject"))
  6512. sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject"))
  6513. // c.TraceOn(os.Stderr)
  6514. testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
  6515. }
  6516. // Test encrypted copy object
  6517. func testEncryptedSSECToSSES3CopyObject() {
  6518. // initialize logging params
  6519. startTime := time.Now()
  6520. testName := getFuncName()
  6521. function := "CopyObject(destination, source)"
  6522. args := map[string]interface{}{}
  6523. // Instantiate new minio client object
  6524. c, err := minio.New(os.Getenv(serverEndpoint),
  6525. &minio.Options{
  6526. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  6527. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  6528. })
  6529. if err != nil {
  6530. logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
  6531. return
  6532. }
  6533. // Generate a new random bucket name.
  6534. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  6535. sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject"))
  6536. sseDst := encrypt.NewSSE()
  6537. // c.TraceOn(os.Stderr)
  6538. testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
  6539. }
  6540. // Test encrypted copy object
  6541. func testEncryptedSSECToUnencryptedCopyObject() {
  6542. // initialize logging params
  6543. startTime := time.Now()
  6544. testName := getFuncName()
  6545. function := "CopyObject(destination, source)"
  6546. args := map[string]interface{}{}
  6547. // Instantiate new minio client object
  6548. c, err := minio.New(os.Getenv(serverEndpoint),
  6549. &minio.Options{
  6550. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  6551. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  6552. })
  6553. if err != nil {
  6554. logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
  6555. return
  6556. }
  6557. // Generate a new random bucket name.
  6558. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  6559. sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject"))
  6560. var sseDst encrypt.ServerSide
  6561. // c.TraceOn(os.Stderr)
  6562. testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
  6563. }
  6564. // Test encrypted copy object
  6565. func testEncryptedSSES3ToSSECCopyObject() {
  6566. // initialize logging params
  6567. startTime := time.Now()
  6568. testName := getFuncName()
  6569. function := "CopyObject(destination, source)"
  6570. args := map[string]interface{}{}
  6571. // Instantiate new minio client object
  6572. c, err := minio.New(os.Getenv(serverEndpoint),
  6573. &minio.Options{
  6574. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  6575. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  6576. })
  6577. if err != nil {
  6578. logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
  6579. return
  6580. }
  6581. // Generate a new random bucket name.
  6582. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  6583. sseSrc := encrypt.NewSSE()
  6584. sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject"))
  6585. // c.TraceOn(os.Stderr)
  6586. testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
  6587. }
  6588. // Test encrypted copy object
  6589. func testEncryptedSSES3ToSSES3CopyObject() {
  6590. // initialize logging params
  6591. startTime := time.Now()
  6592. testName := getFuncName()
  6593. function := "CopyObject(destination, source)"
  6594. args := map[string]interface{}{}
  6595. // Instantiate new minio client object
  6596. c, err := minio.New(os.Getenv(serverEndpoint),
  6597. &minio.Options{
  6598. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  6599. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  6600. })
  6601. if err != nil {
  6602. logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
  6603. return
  6604. }
  6605. // Generate a new random bucket name.
  6606. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  6607. sseSrc := encrypt.NewSSE()
  6608. sseDst := encrypt.NewSSE()
  6609. // c.TraceOn(os.Stderr)
  6610. testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
  6611. }
  6612. // Test encrypted copy object
  6613. func testEncryptedSSES3ToUnencryptedCopyObject() {
  6614. // initialize logging params
  6615. startTime := time.Now()
  6616. testName := getFuncName()
  6617. function := "CopyObject(destination, source)"
  6618. args := map[string]interface{}{}
  6619. // Instantiate new minio client object
  6620. c, err := minio.New(os.Getenv(serverEndpoint),
  6621. &minio.Options{
  6622. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  6623. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  6624. })
  6625. if err != nil {
  6626. logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
  6627. return
  6628. }
  6629. // Generate a new random bucket name.
  6630. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  6631. sseSrc := encrypt.NewSSE()
  6632. var sseDst encrypt.ServerSide
  6633. // c.TraceOn(os.Stderr)
  6634. testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
  6635. }
  6636. // Test encrypted copy object
  6637. func testEncryptedCopyObjectV2() {
  6638. // initialize logging params
  6639. startTime := time.Now()
  6640. testName := getFuncName()
  6641. function := "CopyObject(destination, source)"
  6642. args := map[string]interface{}{}
  6643. // Instantiate new minio client object
  6644. c, err := minio.New(os.Getenv(serverEndpoint),
  6645. &minio.Options{
  6646. Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  6647. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  6648. })
  6649. if err != nil {
  6650. logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
  6651. return
  6652. }
  6653. // Generate a new random bucket name.
  6654. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  6655. sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject"))
  6656. sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject"))
  6657. // c.TraceOn(os.Stderr)
  6658. testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
  6659. }
  6660. func testDecryptedCopyObject() {
  6661. // initialize logging params
  6662. startTime := time.Now()
  6663. testName := getFuncName()
  6664. function := "CopyObject(destination, source)"
  6665. args := map[string]interface{}{}
  6666. // Instantiate new minio client object
  6667. c, err := minio.New(os.Getenv(serverEndpoint),
  6668. &minio.Options{
  6669. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  6670. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  6671. })
  6672. if err != nil {
  6673. logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
  6674. return
  6675. }
  6676. bucketName, objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-"), "object"
  6677. if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}); err != nil {
  6678. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  6679. return
  6680. }
  6681. encryption := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName))
  6682. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(bytes.Repeat([]byte("a"), 1024*1024)), 1024*1024, minio.PutObjectOptions{
  6683. ServerSideEncryption: encryption,
  6684. })
  6685. if err != nil {
  6686. logError(testName, function, args, startTime, "", "PutObject call failed", err)
  6687. return
  6688. }
  6689. src := minio.CopySrcOptions{
  6690. Bucket: bucketName,
  6691. Object: objectName,
  6692. Encryption: encrypt.SSECopy(encryption),
  6693. }
  6694. args["source"] = src
  6695. dst := minio.CopyDestOptions{
  6696. Bucket: bucketName,
  6697. Object: "decrypted-" + objectName,
  6698. }
  6699. args["destination"] = dst
  6700. if _, err = c.CopyObject(context.Background(), dst, src); err != nil {
  6701. logError(testName, function, args, startTime, "", "CopyObject failed", err)
  6702. return
  6703. }
  6704. if _, err = c.GetObject(context.Background(), bucketName, "decrypted-"+objectName, minio.GetObjectOptions{}); err != nil {
  6705. logError(testName, function, args, startTime, "", "GetObject failed", err)
  6706. return
  6707. }
  6708. successLogger(testName, function, args, startTime).Info()
  6709. }
  6710. func testSSECMultipartEncryptedToSSECCopyObjectPart() {
  6711. // initialize logging params
  6712. startTime := time.Now()
  6713. testName := getFuncName()
  6714. function := "CopyObjectPart(destination, source)"
  6715. args := map[string]interface{}{}
  6716. // Instantiate new minio client object
  6717. client, err := minio.New(os.Getenv(serverEndpoint),
  6718. &minio.Options{
  6719. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  6720. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  6721. })
  6722. if err != nil {
  6723. logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
  6724. return
  6725. }
  6726. // Instantiate new core client object.
  6727. c := minio.Core{client}
  6728. // Enable tracing, write to stderr.
  6729. // c.TraceOn(os.Stderr)
  6730. // Set user agent.
  6731. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  6732. // Generate a new random bucket name.
  6733. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
  6734. // Make a new bucket.
  6735. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  6736. if err != nil {
  6737. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  6738. return
  6739. }
  6740. defer cleanupBucket(bucketName, client)
  6741. // Make a buffer with 6MB of data
  6742. buf := bytes.Repeat([]byte("abcdef"), 1024*1024)
  6743. // Save the data
  6744. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  6745. password := "correct horse battery staple"
  6746. srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName))
  6747. // Upload a 6MB object using multipart mechanism
  6748. uploadID, err := c.NewMultipartUpload(context.Background(), bucketName, objectName, minio.PutObjectOptions{ServerSideEncryption: srcencryption})
  6749. if err != nil {
  6750. logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
  6751. return
  6752. }
  6753. var completeParts []minio.CompletePart
  6754. part, err := c.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, bytes.NewReader(buf[:5*1024*1024]), 5*1024*1024, "", "", srcencryption)
  6755. if err != nil {
  6756. logError(testName, function, args, startTime, "", "PutObjectPart call failed", err)
  6757. return
  6758. }
  6759. completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag})
  6760. part, err = c.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 2, bytes.NewReader(buf[5*1024*1024:]), 1024*1024, "", "", srcencryption)
  6761. if err != nil {
  6762. logError(testName, function, args, startTime, "", "PutObjectPart call failed", err)
  6763. return
  6764. }
  6765. completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag})
  6766. // Complete the multipart upload
  6767. _, err = c.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, completeParts, minio.PutObjectOptions{})
  6768. if err != nil {
  6769. logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
  6770. return
  6771. }
  6772. // Stat the object and check its length matches
  6773. objInfo, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption})
  6774. if err != nil {
  6775. logError(testName, function, args, startTime, "", "StatObject call failed", err)
  6776. return
  6777. }
  6778. destBucketName := bucketName
  6779. destObjectName := objectName + "-dest"
  6780. dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName))
  6781. uploadID, err = c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
  6782. if err != nil {
  6783. logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
  6784. return
  6785. }
  6786. // Content of the destination object will be two copies of
  6787. // `objectName` concatenated, followed by first byte of
  6788. // `objectName`.
  6789. metadata := make(map[string]string)
  6790. header := make(http.Header)
  6791. encrypt.SSECopy(srcencryption).Marshal(header)
  6792. dstencryption.Marshal(header)
  6793. for k, v := range header {
  6794. metadata[k] = v[0]
  6795. }
  6796. metadata["x-amz-copy-source-if-match"] = objInfo.ETag
  6797. // First of three parts
  6798. fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
  6799. if err != nil {
  6800. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  6801. return
  6802. }
  6803. // Second of three parts
  6804. sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
  6805. if err != nil {
  6806. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  6807. return
  6808. }
  6809. // Last of three parts
  6810. lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
  6811. if err != nil {
  6812. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  6813. return
  6814. }
  6815. // Complete the multipart upload
  6816. _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
  6817. if err != nil {
  6818. logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
  6819. return
  6820. }
  6821. // Stat the object and check its length matches
  6822. objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption})
  6823. if err != nil {
  6824. logError(testName, function, args, startTime, "", "StatObject call failed", err)
  6825. return
  6826. }
  6827. if objInfo.Size != (6*1024*1024)*2+1 {
  6828. logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
  6829. return
  6830. }
  6831. // Now we read the data back
  6832. getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption}
  6833. getOpts.SetRange(0, 6*1024*1024-1)
  6834. r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
  6835. if err != nil {
  6836. logError(testName, function, args, startTime, "", "GetObject call failed", err)
  6837. return
  6838. }
  6839. getBuf := make([]byte, 6*1024*1024)
  6840. _, err = readFull(r, getBuf)
  6841. if err != nil {
  6842. logError(testName, function, args, startTime, "", "Read buffer failed", err)
  6843. return
  6844. }
  6845. if !bytes.Equal(getBuf, buf) {
  6846. logError(testName, function, args, startTime, "", "Got unexpected data in first 6MB", err)
  6847. return
  6848. }
  6849. getOpts.SetRange(6*1024*1024, 0)
  6850. r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
  6851. if err != nil {
  6852. logError(testName, function, args, startTime, "", "GetObject call failed", err)
  6853. return
  6854. }
  6855. getBuf = make([]byte, 6*1024*1024+1)
  6856. _, err = readFull(r, getBuf)
  6857. if err != nil {
  6858. logError(testName, function, args, startTime, "", "Read buffer failed", err)
  6859. return
  6860. }
  6861. if !bytes.Equal(getBuf[:6*1024*1024], buf) {
  6862. logError(testName, function, args, startTime, "", "Got unexpected data in second 6MB", err)
  6863. return
  6864. }
  6865. if getBuf[6*1024*1024] != buf[0] {
  6866. logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
  6867. return
  6868. }
  6869. successLogger(testName, function, args, startTime).Info()
  6870. // Do not need to remove destBucketName its same as bucketName.
  6871. }
  6872. // Test Core CopyObjectPart implementation
  6873. func testSSECEncryptedToSSECCopyObjectPart() {
  6874. // initialize logging params
  6875. startTime := time.Now()
  6876. testName := getFuncName()
  6877. function := "CopyObjectPart(destination, source)"
  6878. args := map[string]interface{}{}
  6879. // Instantiate new minio client object
  6880. client, err := minio.New(os.Getenv(serverEndpoint),
  6881. &minio.Options{
  6882. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  6883. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  6884. })
  6885. if err != nil {
  6886. logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
  6887. return
  6888. }
  6889. // Instantiate new core client object.
  6890. c := minio.Core{client}
  6891. // Enable tracing, write to stderr.
  6892. // c.TraceOn(os.Stderr)
  6893. // Set user agent.
  6894. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  6895. // Generate a new random bucket name.
  6896. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
  6897. // Make a new bucket.
  6898. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  6899. if err != nil {
  6900. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  6901. return
  6902. }
  6903. defer cleanupBucket(bucketName, client)
  6904. // Make a buffer with 5MB of data
  6905. buf := bytes.Repeat([]byte("abcde"), 1024*1024)
  6906. // Save the data
  6907. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  6908. password := "correct horse battery staple"
  6909. srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName))
  6910. putmetadata := map[string]string{
  6911. "Content-Type": "binary/octet-stream",
  6912. }
  6913. opts := minio.PutObjectOptions{
  6914. UserMetadata: putmetadata,
  6915. ServerSideEncryption: srcencryption,
  6916. }
  6917. uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
  6918. if err != nil {
  6919. logError(testName, function, args, startTime, "", "PutObject call failed", err)
  6920. return
  6921. }
  6922. st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption})
  6923. if err != nil {
  6924. logError(testName, function, args, startTime, "", "StatObject call failed", err)
  6925. return
  6926. }
  6927. if st.Size != int64(len(buf)) {
  6928. logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
  6929. return
  6930. }
  6931. destBucketName := bucketName
  6932. destObjectName := objectName + "-dest"
  6933. dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName))
  6934. uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
  6935. if err != nil {
  6936. logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
  6937. return
  6938. }
  6939. // Content of the destination object will be two copies of
  6940. // `objectName` concatenated, followed by first byte of
  6941. // `objectName`.
  6942. metadata := make(map[string]string)
  6943. header := make(http.Header)
  6944. encrypt.SSECopy(srcencryption).Marshal(header)
  6945. dstencryption.Marshal(header)
  6946. for k, v := range header {
  6947. metadata[k] = v[0]
  6948. }
  6949. metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
  6950. // First of three parts
  6951. fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
  6952. if err != nil {
  6953. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  6954. return
  6955. }
  6956. // Second of three parts
  6957. sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
  6958. if err != nil {
  6959. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  6960. return
  6961. }
  6962. // Last of three parts
  6963. lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
  6964. if err != nil {
  6965. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  6966. return
  6967. }
  6968. // Complete the multipart upload
  6969. _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
  6970. if err != nil {
  6971. logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
  6972. return
  6973. }
  6974. // Stat the object and check its length matches
  6975. objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption})
  6976. if err != nil {
  6977. logError(testName, function, args, startTime, "", "StatObject call failed", err)
  6978. return
  6979. }
  6980. if objInfo.Size != (5*1024*1024)*2+1 {
  6981. logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
  6982. return
  6983. }
  6984. // Now we read the data back
  6985. getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption}
  6986. getOpts.SetRange(0, 5*1024*1024-1)
  6987. r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
  6988. if err != nil {
  6989. logError(testName, function, args, startTime, "", "GetObject call failed", err)
  6990. return
  6991. }
  6992. getBuf := make([]byte, 5*1024*1024)
  6993. _, err = readFull(r, getBuf)
  6994. if err != nil {
  6995. logError(testName, function, args, startTime, "", "Read buffer failed", err)
  6996. return
  6997. }
  6998. if !bytes.Equal(getBuf, buf) {
  6999. logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
  7000. return
  7001. }
  7002. getOpts.SetRange(5*1024*1024, 0)
  7003. r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
  7004. if err != nil {
  7005. logError(testName, function, args, startTime, "", "GetObject call failed", err)
  7006. return
  7007. }
  7008. getBuf = make([]byte, 5*1024*1024+1)
  7009. _, err = readFull(r, getBuf)
  7010. if err != nil {
  7011. logError(testName, function, args, startTime, "", "Read buffer failed", err)
  7012. return
  7013. }
  7014. if !bytes.Equal(getBuf[:5*1024*1024], buf) {
  7015. logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
  7016. return
  7017. }
  7018. if getBuf[5*1024*1024] != buf[0] {
  7019. logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
  7020. return
  7021. }
  7022. successLogger(testName, function, args, startTime).Info()
  7023. // Do not need to remove destBucketName its same as bucketName.
  7024. }
  7025. // Test Core CopyObjectPart implementation for SSEC encrypted to unencrypted copy
  7026. func testSSECEncryptedToUnencryptedCopyPart() {
  7027. // initialize logging params
  7028. startTime := time.Now()
  7029. testName := getFuncName()
  7030. function := "CopyObjectPart(destination, source)"
  7031. args := map[string]interface{}{}
  7032. // Instantiate new minio client object
  7033. client, err := minio.New(os.Getenv(serverEndpoint),
  7034. &minio.Options{
  7035. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  7036. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  7037. })
  7038. if err != nil {
  7039. logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
  7040. return
  7041. }
  7042. // Instantiate new core client object.
  7043. c := minio.Core{client}
  7044. // Enable tracing, write to stderr.
  7045. // c.TraceOn(os.Stderr)
  7046. // Set user agent.
  7047. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  7048. // Generate a new random bucket name.
  7049. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
  7050. // Make a new bucket.
  7051. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  7052. if err != nil {
  7053. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  7054. return
  7055. }
  7056. defer cleanupBucket(bucketName, client)
  7057. // Make a buffer with 5MB of data
  7058. buf := bytes.Repeat([]byte("abcde"), 1024*1024)
  7059. // Save the data
  7060. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  7061. password := "correct horse battery staple"
  7062. srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName))
  7063. opts := minio.PutObjectOptions{
  7064. UserMetadata: map[string]string{
  7065. "Content-Type": "binary/octet-stream",
  7066. },
  7067. ServerSideEncryption: srcencryption,
  7068. }
  7069. uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
  7070. if err != nil {
  7071. logError(testName, function, args, startTime, "", "PutObject call failed", err)
  7072. return
  7073. }
  7074. st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption})
  7075. if err != nil {
  7076. logError(testName, function, args, startTime, "", "StatObject call failed", err)
  7077. return
  7078. }
  7079. if st.Size != int64(len(buf)) {
  7080. logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
  7081. return
  7082. }
  7083. destBucketName := bucketName
  7084. destObjectName := objectName + "-dest"
  7085. var dstencryption encrypt.ServerSide
  7086. uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
  7087. if err != nil {
  7088. logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
  7089. return
  7090. }
  7091. // Content of the destination object will be two copies of
  7092. // `objectName` concatenated, followed by first byte of
  7093. // `objectName`.
  7094. metadata := make(map[string]string)
  7095. header := make(http.Header)
  7096. encrypt.SSECopy(srcencryption).Marshal(header)
  7097. for k, v := range header {
  7098. metadata[k] = v[0]
  7099. }
  7100. metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
  7101. // First of three parts
  7102. fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
  7103. if err != nil {
  7104. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  7105. return
  7106. }
  7107. // Second of three parts
  7108. sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
  7109. if err != nil {
  7110. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  7111. return
  7112. }
  7113. // Last of three parts
  7114. lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
  7115. if err != nil {
  7116. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  7117. return
  7118. }
  7119. // Complete the multipart upload
  7120. _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
  7121. if err != nil {
  7122. logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
  7123. return
  7124. }
  7125. // Stat the object and check its length matches
  7126. objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{})
  7127. if err != nil {
  7128. logError(testName, function, args, startTime, "", "StatObject call failed", err)
  7129. return
  7130. }
  7131. if objInfo.Size != (5*1024*1024)*2+1 {
  7132. logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
  7133. return
  7134. }
  7135. // Now we read the data back
  7136. getOpts := minio.GetObjectOptions{}
  7137. getOpts.SetRange(0, 5*1024*1024-1)
  7138. r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
  7139. if err != nil {
  7140. logError(testName, function, args, startTime, "", "GetObject call failed", err)
  7141. return
  7142. }
  7143. getBuf := make([]byte, 5*1024*1024)
  7144. _, err = readFull(r, getBuf)
  7145. if err != nil {
  7146. logError(testName, function, args, startTime, "", "Read buffer failed", err)
  7147. return
  7148. }
  7149. if !bytes.Equal(getBuf, buf) {
  7150. logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
  7151. return
  7152. }
  7153. getOpts.SetRange(5*1024*1024, 0)
  7154. r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
  7155. if err != nil {
  7156. logError(testName, function, args, startTime, "", "GetObject call failed", err)
  7157. return
  7158. }
  7159. getBuf = make([]byte, 5*1024*1024+1)
  7160. _, err = readFull(r, getBuf)
  7161. if err != nil {
  7162. logError(testName, function, args, startTime, "", "Read buffer failed", err)
  7163. return
  7164. }
  7165. if !bytes.Equal(getBuf[:5*1024*1024], buf) {
  7166. logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
  7167. return
  7168. }
  7169. if getBuf[5*1024*1024] != buf[0] {
  7170. logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
  7171. return
  7172. }
  7173. successLogger(testName, function, args, startTime).Info()
  7174. // Do not need to remove destBucketName its same as bucketName.
  7175. }
  7176. // Test Core CopyObjectPart implementation for SSEC encrypted to SSE-S3 encrypted copy
  7177. func testSSECEncryptedToSSES3CopyObjectPart() {
  7178. // initialize logging params
  7179. startTime := time.Now()
  7180. testName := getFuncName()
  7181. function := "CopyObjectPart(destination, source)"
  7182. args := map[string]interface{}{}
  7183. // Instantiate new minio client object
  7184. client, err := minio.New(os.Getenv(serverEndpoint),
  7185. &minio.Options{
  7186. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  7187. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  7188. })
  7189. if err != nil {
  7190. logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
  7191. return
  7192. }
  7193. // Instantiate new core client object.
  7194. c := minio.Core{client}
  7195. // Enable tracing, write to stderr.
  7196. // c.TraceOn(os.Stderr)
  7197. // Set user agent.
  7198. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  7199. // Generate a new random bucket name.
  7200. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
  7201. // Make a new bucket.
  7202. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  7203. if err != nil {
  7204. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  7205. return
  7206. }
  7207. defer cleanupBucket(bucketName, client)
  7208. // Make a buffer with 5MB of data
  7209. buf := bytes.Repeat([]byte("abcde"), 1024*1024)
  7210. // Save the data
  7211. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  7212. password := "correct horse battery staple"
  7213. srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName))
  7214. putmetadata := map[string]string{
  7215. "Content-Type": "binary/octet-stream",
  7216. }
  7217. opts := minio.PutObjectOptions{
  7218. UserMetadata: putmetadata,
  7219. ServerSideEncryption: srcencryption,
  7220. }
  7221. uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
  7222. if err != nil {
  7223. logError(testName, function, args, startTime, "", "PutObject call failed", err)
  7224. return
  7225. }
  7226. st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption})
  7227. if err != nil {
  7228. logError(testName, function, args, startTime, "", "StatObject call failed", err)
  7229. return
  7230. }
  7231. if st.Size != int64(len(buf)) {
  7232. logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
  7233. return
  7234. }
  7235. destBucketName := bucketName
  7236. destObjectName := objectName + "-dest"
  7237. dstencryption := encrypt.NewSSE()
  7238. uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
  7239. if err != nil {
  7240. logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
  7241. return
  7242. }
  7243. // Content of the destination object will be two copies of
  7244. // `objectName` concatenated, followed by first byte of
  7245. // `objectName`.
  7246. metadata := make(map[string]string)
  7247. header := make(http.Header)
  7248. encrypt.SSECopy(srcencryption).Marshal(header)
  7249. dstencryption.Marshal(header)
  7250. for k, v := range header {
  7251. metadata[k] = v[0]
  7252. }
  7253. metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
  7254. // First of three parts
  7255. fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
  7256. if err != nil {
  7257. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  7258. return
  7259. }
  7260. // Second of three parts
  7261. sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
  7262. if err != nil {
  7263. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  7264. return
  7265. }
  7266. // Last of three parts
  7267. lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
  7268. if err != nil {
  7269. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  7270. return
  7271. }
  7272. // Complete the multipart upload
  7273. _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
  7274. if err != nil {
  7275. logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
  7276. return
  7277. }
  7278. // Stat the object and check its length matches
  7279. objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{})
  7280. if err != nil {
  7281. logError(testName, function, args, startTime, "", "StatObject call failed", err)
  7282. return
  7283. }
  7284. if objInfo.Size != (5*1024*1024)*2+1 {
  7285. logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
  7286. return
  7287. }
  7288. // Now we read the data back
  7289. getOpts := minio.GetObjectOptions{}
  7290. getOpts.SetRange(0, 5*1024*1024-1)
  7291. r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
  7292. if err != nil {
  7293. logError(testName, function, args, startTime, "", "GetObject call failed", err)
  7294. return
  7295. }
  7296. getBuf := make([]byte, 5*1024*1024)
  7297. _, err = readFull(r, getBuf)
  7298. if err != nil {
  7299. logError(testName, function, args, startTime, "", "Read buffer failed", err)
  7300. return
  7301. }
  7302. if !bytes.Equal(getBuf, buf) {
  7303. logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
  7304. return
  7305. }
  7306. getOpts.SetRange(5*1024*1024, 0)
  7307. r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
  7308. if err != nil {
  7309. logError(testName, function, args, startTime, "", "GetObject call failed", err)
  7310. return
  7311. }
  7312. getBuf = make([]byte, 5*1024*1024+1)
  7313. _, err = readFull(r, getBuf)
  7314. if err != nil {
  7315. logError(testName, function, args, startTime, "", "Read buffer failed", err)
  7316. return
  7317. }
  7318. if !bytes.Equal(getBuf[:5*1024*1024], buf) {
  7319. logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
  7320. return
  7321. }
  7322. if getBuf[5*1024*1024] != buf[0] {
  7323. logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
  7324. return
  7325. }
  7326. successLogger(testName, function, args, startTime).Info()
  7327. // Do not need to remove destBucketName its same as bucketName.
  7328. }
  7329. // Test Core CopyObjectPart implementation for unencrypted to SSEC encryption copy part
  7330. func testUnencryptedToSSECCopyObjectPart() {
  7331. // initialize logging params
  7332. startTime := time.Now()
  7333. testName := getFuncName()
  7334. function := "CopyObjectPart(destination, source)"
  7335. args := map[string]interface{}{}
  7336. // Instantiate new minio client object
  7337. client, err := minio.New(os.Getenv(serverEndpoint),
  7338. &minio.Options{
  7339. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  7340. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  7341. })
  7342. if err != nil {
  7343. logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
  7344. return
  7345. }
  7346. // Instantiate new core client object.
  7347. c := minio.Core{client}
  7348. // Enable tracing, write to stderr.
  7349. // c.TraceOn(os.Stderr)
  7350. // Set user agent.
  7351. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  7352. // Generate a new random bucket name.
  7353. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
  7354. // Make a new bucket.
  7355. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  7356. if err != nil {
  7357. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  7358. return
  7359. }
  7360. defer cleanupBucket(bucketName, client)
  7361. // Make a buffer with 5MB of data
  7362. buf := bytes.Repeat([]byte("abcde"), 1024*1024)
  7363. // Save the data
  7364. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  7365. password := "correct horse battery staple"
  7366. putmetadata := map[string]string{
  7367. "Content-Type": "binary/octet-stream",
  7368. }
  7369. opts := minio.PutObjectOptions{
  7370. UserMetadata: putmetadata,
  7371. }
  7372. uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
  7373. if err != nil {
  7374. logError(testName, function, args, startTime, "", "PutObject call failed", err)
  7375. return
  7376. }
  7377. st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
  7378. if err != nil {
  7379. logError(testName, function, args, startTime, "", "StatObject call failed", err)
  7380. return
  7381. }
  7382. if st.Size != int64(len(buf)) {
  7383. logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
  7384. return
  7385. }
  7386. destBucketName := bucketName
  7387. destObjectName := objectName + "-dest"
  7388. dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName))
  7389. uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
  7390. if err != nil {
  7391. logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
  7392. return
  7393. }
  7394. // Content of the destination object will be two copies of
  7395. // `objectName` concatenated, followed by first byte of
  7396. // `objectName`.
  7397. metadata := make(map[string]string)
  7398. header := make(http.Header)
  7399. dstencryption.Marshal(header)
  7400. for k, v := range header {
  7401. metadata[k] = v[0]
  7402. }
  7403. metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
  7404. // First of three parts
  7405. fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
  7406. if err != nil {
  7407. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  7408. return
  7409. }
  7410. // Second of three parts
  7411. sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
  7412. if err != nil {
  7413. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  7414. return
  7415. }
  7416. // Last of three parts
  7417. lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
  7418. if err != nil {
  7419. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  7420. return
  7421. }
  7422. // Complete the multipart upload
  7423. _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
  7424. if err != nil {
  7425. logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
  7426. return
  7427. }
  7428. // Stat the object and check its length matches
  7429. objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption})
  7430. if err != nil {
  7431. logError(testName, function, args, startTime, "", "StatObject call failed", err)
  7432. return
  7433. }
  7434. if objInfo.Size != (5*1024*1024)*2+1 {
  7435. logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
  7436. return
  7437. }
  7438. // Now we read the data back
  7439. getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption}
  7440. getOpts.SetRange(0, 5*1024*1024-1)
  7441. r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
  7442. if err != nil {
  7443. logError(testName, function, args, startTime, "", "GetObject call failed", err)
  7444. return
  7445. }
  7446. getBuf := make([]byte, 5*1024*1024)
  7447. _, err = readFull(r, getBuf)
  7448. if err != nil {
  7449. logError(testName, function, args, startTime, "", "Read buffer failed", err)
  7450. return
  7451. }
  7452. if !bytes.Equal(getBuf, buf) {
  7453. logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
  7454. return
  7455. }
  7456. getOpts.SetRange(5*1024*1024, 0)
  7457. r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
  7458. if err != nil {
  7459. logError(testName, function, args, startTime, "", "GetObject call failed", err)
  7460. return
  7461. }
  7462. getBuf = make([]byte, 5*1024*1024+1)
  7463. _, err = readFull(r, getBuf)
  7464. if err != nil {
  7465. logError(testName, function, args, startTime, "", "Read buffer failed", err)
  7466. return
  7467. }
  7468. if !bytes.Equal(getBuf[:5*1024*1024], buf) {
  7469. logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
  7470. return
  7471. }
  7472. if getBuf[5*1024*1024] != buf[0] {
  7473. logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
  7474. return
  7475. }
  7476. successLogger(testName, function, args, startTime).Info()
  7477. // Do not need to remove destBucketName its same as bucketName.
  7478. }
  7479. // Test Core CopyObjectPart implementation for unencrypted to unencrypted copy
  7480. func testUnencryptedToUnencryptedCopyPart() {
  7481. // initialize logging params
  7482. startTime := time.Now()
  7483. testName := getFuncName()
  7484. function := "CopyObjectPart(destination, source)"
  7485. args := map[string]interface{}{}
  7486. // Instantiate new minio client object
  7487. client, err := minio.New(os.Getenv(serverEndpoint),
  7488. &minio.Options{
  7489. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  7490. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  7491. })
  7492. if err != nil {
  7493. logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
  7494. return
  7495. }
  7496. // Instantiate new core client object.
  7497. c := minio.Core{client}
  7498. // Enable tracing, write to stderr.
  7499. // c.TraceOn(os.Stderr)
  7500. // Set user agent.
  7501. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  7502. // Generate a new random bucket name.
  7503. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
  7504. // Make a new bucket.
  7505. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  7506. if err != nil {
  7507. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  7508. return
  7509. }
  7510. defer cleanupBucket(bucketName, client)
  7511. // Make a buffer with 5MB of data
  7512. buf := bytes.Repeat([]byte("abcde"), 1024*1024)
  7513. // Save the data
  7514. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  7515. putmetadata := map[string]string{
  7516. "Content-Type": "binary/octet-stream",
  7517. }
  7518. opts := minio.PutObjectOptions{
  7519. UserMetadata: putmetadata,
  7520. }
  7521. uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
  7522. if err != nil {
  7523. logError(testName, function, args, startTime, "", "PutObject call failed", err)
  7524. return
  7525. }
  7526. st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
  7527. if err != nil {
  7528. logError(testName, function, args, startTime, "", "StatObject call failed", err)
  7529. return
  7530. }
  7531. if st.Size != int64(len(buf)) {
  7532. logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
  7533. return
  7534. }
  7535. destBucketName := bucketName
  7536. destObjectName := objectName + "-dest"
  7537. uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{})
  7538. if err != nil {
  7539. logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
  7540. return
  7541. }
  7542. // Content of the destination object will be two copies of
  7543. // `objectName` concatenated, followed by first byte of
  7544. // `objectName`.
  7545. metadata := make(map[string]string)
  7546. header := make(http.Header)
  7547. for k, v := range header {
  7548. metadata[k] = v[0]
  7549. }
  7550. metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
  7551. // First of three parts
  7552. fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
  7553. if err != nil {
  7554. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  7555. return
  7556. }
  7557. // Second of three parts
  7558. sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
  7559. if err != nil {
  7560. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  7561. return
  7562. }
  7563. // Last of three parts
  7564. lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
  7565. if err != nil {
  7566. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  7567. return
  7568. }
  7569. // Complete the multipart upload
  7570. _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
  7571. if err != nil {
  7572. logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
  7573. return
  7574. }
  7575. // Stat the object and check its length matches
  7576. objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{})
  7577. if err != nil {
  7578. logError(testName, function, args, startTime, "", "StatObject call failed", err)
  7579. return
  7580. }
  7581. if objInfo.Size != (5*1024*1024)*2+1 {
  7582. logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
  7583. return
  7584. }
  7585. // Now we read the data back
  7586. getOpts := minio.GetObjectOptions{}
  7587. getOpts.SetRange(0, 5*1024*1024-1)
  7588. r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
  7589. if err != nil {
  7590. logError(testName, function, args, startTime, "", "GetObject call failed", err)
  7591. return
  7592. }
  7593. getBuf := make([]byte, 5*1024*1024)
  7594. _, err = readFull(r, getBuf)
  7595. if err != nil {
  7596. logError(testName, function, args, startTime, "", "Read buffer failed", err)
  7597. return
  7598. }
  7599. if !bytes.Equal(getBuf, buf) {
  7600. logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
  7601. return
  7602. }
  7603. getOpts.SetRange(5*1024*1024, 0)
  7604. r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
  7605. if err != nil {
  7606. logError(testName, function, args, startTime, "", "GetObject call failed", err)
  7607. return
  7608. }
  7609. getBuf = make([]byte, 5*1024*1024+1)
  7610. _, err = readFull(r, getBuf)
  7611. if err != nil {
  7612. logError(testName, function, args, startTime, "", "Read buffer failed", err)
  7613. return
  7614. }
  7615. if !bytes.Equal(getBuf[:5*1024*1024], buf) {
  7616. logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
  7617. return
  7618. }
  7619. if getBuf[5*1024*1024] != buf[0] {
  7620. logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
  7621. return
  7622. }
  7623. successLogger(testName, function, args, startTime).Info()
  7624. // Do not need to remove destBucketName its same as bucketName.
  7625. }
  7626. // Test Core CopyObjectPart implementation for unencrypted to SSE-S3 encrypted copy
  7627. func testUnencryptedToSSES3CopyObjectPart() {
  7628. // initialize logging params
  7629. startTime := time.Now()
  7630. testName := getFuncName()
  7631. function := "CopyObjectPart(destination, source)"
  7632. args := map[string]interface{}{}
  7633. // Instantiate new minio client object
  7634. client, err := minio.New(os.Getenv(serverEndpoint),
  7635. &minio.Options{
  7636. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  7637. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  7638. })
  7639. if err != nil {
  7640. logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
  7641. return
  7642. }
  7643. // Instantiate new core client object.
  7644. c := minio.Core{client}
  7645. // Enable tracing, write to stderr.
  7646. // c.TraceOn(os.Stderr)
  7647. // Set user agent.
  7648. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  7649. // Generate a new random bucket name.
  7650. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
  7651. // Make a new bucket.
  7652. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  7653. if err != nil {
  7654. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  7655. return
  7656. }
  7657. defer cleanupBucket(bucketName, client)
  7658. // Make a buffer with 5MB of data
  7659. buf := bytes.Repeat([]byte("abcde"), 1024*1024)
  7660. // Save the data
  7661. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  7662. opts := minio.PutObjectOptions{
  7663. UserMetadata: map[string]string{
  7664. "Content-Type": "binary/octet-stream",
  7665. },
  7666. }
  7667. uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
  7668. if err != nil {
  7669. logError(testName, function, args, startTime, "", "PutObject call failed", err)
  7670. return
  7671. }
  7672. st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
  7673. if err != nil {
  7674. logError(testName, function, args, startTime, "", "StatObject call failed", err)
  7675. return
  7676. }
  7677. if st.Size != int64(len(buf)) {
  7678. logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
  7679. return
  7680. }
  7681. destBucketName := bucketName
  7682. destObjectName := objectName + "-dest"
  7683. dstencryption := encrypt.NewSSE()
  7684. uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
  7685. if err != nil {
  7686. logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
  7687. return
  7688. }
  7689. // Content of the destination object will be two copies of
  7690. // `objectName` concatenated, followed by first byte of
  7691. // `objectName`.
  7692. metadata := make(map[string]string)
  7693. header := make(http.Header)
  7694. dstencryption.Marshal(header)
  7695. for k, v := range header {
  7696. metadata[k] = v[0]
  7697. }
  7698. metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
  7699. // First of three parts
  7700. fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
  7701. if err != nil {
  7702. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  7703. return
  7704. }
  7705. // Second of three parts
  7706. sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
  7707. if err != nil {
  7708. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  7709. return
  7710. }
  7711. // Last of three parts
  7712. lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
  7713. if err != nil {
  7714. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  7715. return
  7716. }
  7717. // Complete the multipart upload
  7718. _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
  7719. if err != nil {
  7720. logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
  7721. return
  7722. }
  7723. // Stat the object and check its length matches
  7724. objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{})
  7725. if err != nil {
  7726. logError(testName, function, args, startTime, "", "StatObject call failed", err)
  7727. return
  7728. }
  7729. if objInfo.Size != (5*1024*1024)*2+1 {
  7730. logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
  7731. return
  7732. }
  7733. // Now we read the data back
  7734. getOpts := minio.GetObjectOptions{}
  7735. getOpts.SetRange(0, 5*1024*1024-1)
  7736. r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
  7737. if err != nil {
  7738. logError(testName, function, args, startTime, "", "GetObject call failed", err)
  7739. return
  7740. }
  7741. getBuf := make([]byte, 5*1024*1024)
  7742. _, err = readFull(r, getBuf)
  7743. if err != nil {
  7744. logError(testName, function, args, startTime, "", "Read buffer failed", err)
  7745. return
  7746. }
  7747. if !bytes.Equal(getBuf, buf) {
  7748. logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
  7749. return
  7750. }
  7751. getOpts.SetRange(5*1024*1024, 0)
  7752. r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
  7753. if err != nil {
  7754. logError(testName, function, args, startTime, "", "GetObject call failed", err)
  7755. return
  7756. }
  7757. getBuf = make([]byte, 5*1024*1024+1)
  7758. _, err = readFull(r, getBuf)
  7759. if err != nil {
  7760. logError(testName, function, args, startTime, "", "Read buffer failed", err)
  7761. return
  7762. }
  7763. if !bytes.Equal(getBuf[:5*1024*1024], buf) {
  7764. logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
  7765. return
  7766. }
  7767. if getBuf[5*1024*1024] != buf[0] {
  7768. logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
  7769. return
  7770. }
  7771. successLogger(testName, function, args, startTime).Info()
  7772. // Do not need to remove destBucketName its same as bucketName.
  7773. }
  7774. // Test Core CopyObjectPart implementation for SSE-S3 to SSEC encryption copy part
  7775. func testSSES3EncryptedToSSECCopyObjectPart() {
  7776. // initialize logging params
  7777. startTime := time.Now()
  7778. testName := getFuncName()
  7779. function := "CopyObjectPart(destination, source)"
  7780. args := map[string]interface{}{}
  7781. // Instantiate new minio client object
  7782. client, err := minio.New(os.Getenv(serverEndpoint),
  7783. &minio.Options{
  7784. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  7785. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  7786. })
  7787. if err != nil {
  7788. logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
  7789. return
  7790. }
  7791. // Instantiate new core client object.
  7792. c := minio.Core{client}
  7793. // Enable tracing, write to stderr.
  7794. // c.TraceOn(os.Stderr)
  7795. // Set user agent.
  7796. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  7797. // Generate a new random bucket name.
  7798. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
  7799. // Make a new bucket.
  7800. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  7801. if err != nil {
  7802. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  7803. return
  7804. }
  7805. defer cleanupBucket(bucketName, client)
  7806. // Make a buffer with 5MB of data
  7807. buf := bytes.Repeat([]byte("abcde"), 1024*1024)
  7808. // Save the data
  7809. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  7810. password := "correct horse battery staple"
  7811. srcEncryption := encrypt.NewSSE()
  7812. opts := minio.PutObjectOptions{
  7813. UserMetadata: map[string]string{
  7814. "Content-Type": "binary/octet-stream",
  7815. },
  7816. ServerSideEncryption: srcEncryption,
  7817. }
  7818. uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
  7819. if err != nil {
  7820. logError(testName, function, args, startTime, "", "PutObject call failed", err)
  7821. return
  7822. }
  7823. st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcEncryption})
  7824. if err != nil {
  7825. logError(testName, function, args, startTime, "", "StatObject call failed", err)
  7826. return
  7827. }
  7828. if st.Size != int64(len(buf)) {
  7829. logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
  7830. return
  7831. }
  7832. destBucketName := bucketName
  7833. destObjectName := objectName + "-dest"
  7834. dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName))
  7835. uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
  7836. if err != nil {
  7837. logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
  7838. return
  7839. }
  7840. // Content of the destination object will be two copies of
  7841. // `objectName` concatenated, followed by first byte of
  7842. // `objectName`.
  7843. metadata := make(map[string]string)
  7844. header := make(http.Header)
  7845. dstencryption.Marshal(header)
  7846. for k, v := range header {
  7847. metadata[k] = v[0]
  7848. }
  7849. metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
  7850. // First of three parts
  7851. fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
  7852. if err != nil {
  7853. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  7854. return
  7855. }
  7856. // Second of three parts
  7857. sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
  7858. if err != nil {
  7859. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  7860. return
  7861. }
  7862. // Last of three parts
  7863. lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
  7864. if err != nil {
  7865. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  7866. return
  7867. }
  7868. // Complete the multipart upload
  7869. _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
  7870. if err != nil {
  7871. logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
  7872. return
  7873. }
  7874. // Stat the object and check its length matches
  7875. objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption})
  7876. if err != nil {
  7877. logError(testName, function, args, startTime, "", "StatObject call failed", err)
  7878. return
  7879. }
  7880. if objInfo.Size != (5*1024*1024)*2+1 {
  7881. logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
  7882. return
  7883. }
  7884. // Now we read the data back
  7885. getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption}
  7886. getOpts.SetRange(0, 5*1024*1024-1)
  7887. r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
  7888. if err != nil {
  7889. logError(testName, function, args, startTime, "", "GetObject call failed", err)
  7890. return
  7891. }
  7892. getBuf := make([]byte, 5*1024*1024)
  7893. _, err = readFull(r, getBuf)
  7894. if err != nil {
  7895. logError(testName, function, args, startTime, "", "Read buffer failed", err)
  7896. return
  7897. }
  7898. if !bytes.Equal(getBuf, buf) {
  7899. logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
  7900. return
  7901. }
  7902. getOpts.SetRange(5*1024*1024, 0)
  7903. r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
  7904. if err != nil {
  7905. logError(testName, function, args, startTime, "", "GetObject call failed", err)
  7906. return
  7907. }
  7908. getBuf = make([]byte, 5*1024*1024+1)
  7909. _, err = readFull(r, getBuf)
  7910. if err != nil {
  7911. logError(testName, function, args, startTime, "", "Read buffer failed", err)
  7912. return
  7913. }
  7914. if !bytes.Equal(getBuf[:5*1024*1024], buf) {
  7915. logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
  7916. return
  7917. }
  7918. if getBuf[5*1024*1024] != buf[0] {
  7919. logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
  7920. return
  7921. }
  7922. successLogger(testName, function, args, startTime).Info()
  7923. // Do not need to remove destBucketName its same as bucketName.
  7924. }
  7925. // Test Core CopyObjectPart implementation for unencrypted to unencrypted copy
  7926. func testSSES3EncryptedToUnencryptedCopyPart() {
  7927. // initialize logging params
  7928. startTime := time.Now()
  7929. testName := getFuncName()
  7930. function := "CopyObjectPart(destination, source)"
  7931. args := map[string]interface{}{}
  7932. // Instantiate new minio client object
  7933. client, err := minio.New(os.Getenv(serverEndpoint),
  7934. &minio.Options{
  7935. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  7936. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  7937. })
  7938. if err != nil {
  7939. logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
  7940. return
  7941. }
  7942. // Instantiate new core client object.
  7943. c := minio.Core{client}
  7944. // Enable tracing, write to stderr.
  7945. // c.TraceOn(os.Stderr)
  7946. // Set user agent.
  7947. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  7948. // Generate a new random bucket name.
  7949. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
  7950. // Make a new bucket.
  7951. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  7952. if err != nil {
  7953. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  7954. return
  7955. }
  7956. defer cleanupBucket(bucketName, client)
  7957. // Make a buffer with 5MB of data
  7958. buf := bytes.Repeat([]byte("abcde"), 1024*1024)
  7959. // Save the data
  7960. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  7961. srcEncryption := encrypt.NewSSE()
  7962. opts := minio.PutObjectOptions{
  7963. UserMetadata: map[string]string{
  7964. "Content-Type": "binary/octet-stream",
  7965. },
  7966. ServerSideEncryption: srcEncryption,
  7967. }
  7968. uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
  7969. if err != nil {
  7970. logError(testName, function, args, startTime, "", "PutObject call failed", err)
  7971. return
  7972. }
  7973. st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcEncryption})
  7974. if err != nil {
  7975. logError(testName, function, args, startTime, "", "StatObject call failed", err)
  7976. return
  7977. }
  7978. if st.Size != int64(len(buf)) {
  7979. logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
  7980. return
  7981. }
  7982. destBucketName := bucketName
  7983. destObjectName := objectName + "-dest"
  7984. uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{})
  7985. if err != nil {
  7986. logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
  7987. return
  7988. }
  7989. // Content of the destination object will be two copies of
  7990. // `objectName` concatenated, followed by first byte of
  7991. // `objectName`.
  7992. metadata := make(map[string]string)
  7993. header := make(http.Header)
  7994. for k, v := range header {
  7995. metadata[k] = v[0]
  7996. }
  7997. metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
  7998. // First of three parts
  7999. fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
  8000. if err != nil {
  8001. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  8002. return
  8003. }
  8004. // Second of three parts
  8005. sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
  8006. if err != nil {
  8007. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  8008. return
  8009. }
  8010. // Last of three parts
  8011. lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
  8012. if err != nil {
  8013. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  8014. return
  8015. }
  8016. // Complete the multipart upload
  8017. _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
  8018. if err != nil {
  8019. logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
  8020. return
  8021. }
  8022. // Stat the object and check its length matches
  8023. objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{})
  8024. if err != nil {
  8025. logError(testName, function, args, startTime, "", "StatObject call failed", err)
  8026. return
  8027. }
  8028. if objInfo.Size != (5*1024*1024)*2+1 {
  8029. logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
  8030. return
  8031. }
  8032. // Now we read the data back
  8033. getOpts := minio.GetObjectOptions{}
  8034. getOpts.SetRange(0, 5*1024*1024-1)
  8035. r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
  8036. if err != nil {
  8037. logError(testName, function, args, startTime, "", "GetObject call failed", err)
  8038. return
  8039. }
  8040. getBuf := make([]byte, 5*1024*1024)
  8041. _, err = readFull(r, getBuf)
  8042. if err != nil {
  8043. logError(testName, function, args, startTime, "", "Read buffer failed", err)
  8044. return
  8045. }
  8046. if !bytes.Equal(getBuf, buf) {
  8047. logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
  8048. return
  8049. }
  8050. getOpts.SetRange(5*1024*1024, 0)
  8051. r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
  8052. if err != nil {
  8053. logError(testName, function, args, startTime, "", "GetObject call failed", err)
  8054. return
  8055. }
  8056. getBuf = make([]byte, 5*1024*1024+1)
  8057. _, err = readFull(r, getBuf)
  8058. if err != nil {
  8059. logError(testName, function, args, startTime, "", "Read buffer failed", err)
  8060. return
  8061. }
  8062. if !bytes.Equal(getBuf[:5*1024*1024], buf) {
  8063. logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
  8064. return
  8065. }
  8066. if getBuf[5*1024*1024] != buf[0] {
  8067. logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
  8068. return
  8069. }
  8070. successLogger(testName, function, args, startTime).Info()
  8071. // Do not need to remove destBucketName its same as bucketName.
  8072. }
  8073. // Test Core CopyObjectPart implementation for unencrypted to SSE-S3 encrypted copy
  8074. func testSSES3EncryptedToSSES3CopyObjectPart() {
  8075. // initialize logging params
  8076. startTime := time.Now()
  8077. testName := getFuncName()
  8078. function := "CopyObjectPart(destination, source)"
  8079. args := map[string]interface{}{}
  8080. // Instantiate new minio client object
  8081. client, err := minio.New(os.Getenv(serverEndpoint),
  8082. &minio.Options{
  8083. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  8084. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  8085. })
  8086. if err != nil {
  8087. logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
  8088. return
  8089. }
  8090. // Instantiate new core client object.
  8091. c := minio.Core{client}
  8092. // Enable tracing, write to stderr.
  8093. // c.TraceOn(os.Stderr)
  8094. // Set user agent.
  8095. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  8096. // Generate a new random bucket name.
  8097. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
  8098. // Make a new bucket.
  8099. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  8100. if err != nil {
  8101. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  8102. return
  8103. }
  8104. defer cleanupBucket(bucketName, client)
  8105. // Make a buffer with 5MB of data
  8106. buf := bytes.Repeat([]byte("abcde"), 1024*1024)
  8107. // Save the data
  8108. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  8109. srcEncryption := encrypt.NewSSE()
  8110. opts := minio.PutObjectOptions{
  8111. UserMetadata: map[string]string{
  8112. "Content-Type": "binary/octet-stream",
  8113. },
  8114. ServerSideEncryption: srcEncryption,
  8115. }
  8116. uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
  8117. if err != nil {
  8118. logError(testName, function, args, startTime, "", "PutObject call failed", err)
  8119. return
  8120. }
  8121. st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcEncryption})
  8122. if err != nil {
  8123. logError(testName, function, args, startTime, "", "StatObject call failed", err)
  8124. return
  8125. }
  8126. if st.Size != int64(len(buf)) {
  8127. logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
  8128. return
  8129. }
  8130. destBucketName := bucketName
  8131. destObjectName := objectName + "-dest"
  8132. dstencryption := encrypt.NewSSE()
  8133. uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
  8134. if err != nil {
  8135. logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
  8136. return
  8137. }
  8138. // Content of the destination object will be two copies of
  8139. // `objectName` concatenated, followed by first byte of
  8140. // `objectName`.
  8141. metadata := make(map[string]string)
  8142. header := make(http.Header)
  8143. dstencryption.Marshal(header)
  8144. for k, v := range header {
  8145. metadata[k] = v[0]
  8146. }
  8147. metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
  8148. // First of three parts
  8149. fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
  8150. if err != nil {
  8151. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  8152. return
  8153. }
  8154. // Second of three parts
  8155. sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
  8156. if err != nil {
  8157. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  8158. return
  8159. }
  8160. // Last of three parts
  8161. lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
  8162. if err != nil {
  8163. logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
  8164. return
  8165. }
  8166. // Complete the multipart upload
  8167. _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
  8168. if err != nil {
  8169. logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
  8170. return
  8171. }
  8172. // Stat the object and check its length matches
  8173. objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{})
  8174. if err != nil {
  8175. logError(testName, function, args, startTime, "", "StatObject call failed", err)
  8176. return
  8177. }
  8178. if objInfo.Size != (5*1024*1024)*2+1 {
  8179. logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
  8180. return
  8181. }
  8182. // Now we read the data back
  8183. getOpts := minio.GetObjectOptions{}
  8184. getOpts.SetRange(0, 5*1024*1024-1)
  8185. r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
  8186. if err != nil {
  8187. logError(testName, function, args, startTime, "", "GetObject call failed", err)
  8188. return
  8189. }
  8190. getBuf := make([]byte, 5*1024*1024)
  8191. _, err = readFull(r, getBuf)
  8192. if err != nil {
  8193. logError(testName, function, args, startTime, "", "Read buffer failed", err)
  8194. return
  8195. }
  8196. if !bytes.Equal(getBuf, buf) {
  8197. logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
  8198. return
  8199. }
  8200. getOpts.SetRange(5*1024*1024, 0)
  8201. r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
  8202. if err != nil {
  8203. logError(testName, function, args, startTime, "", "GetObject call failed", err)
  8204. return
  8205. }
  8206. getBuf = make([]byte, 5*1024*1024+1)
  8207. _, err = readFull(r, getBuf)
  8208. if err != nil {
  8209. logError(testName, function, args, startTime, "", "Read buffer failed", err)
  8210. return
  8211. }
  8212. if !bytes.Equal(getBuf[:5*1024*1024], buf) {
  8213. logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
  8214. return
  8215. }
  8216. if getBuf[5*1024*1024] != buf[0] {
  8217. logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
  8218. return
  8219. }
  8220. successLogger(testName, function, args, startTime).Info()
  8221. // Do not need to remove destBucketName its same as bucketName.
  8222. }
  8223. func testUserMetadataCopying() {
  8224. // initialize logging params
  8225. startTime := time.Now()
  8226. testName := getFuncName()
  8227. function := "CopyObject(destination, source)"
  8228. args := map[string]interface{}{}
  8229. // Instantiate new minio client object
  8230. c, err := minio.New(os.Getenv(serverEndpoint),
  8231. &minio.Options{
  8232. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  8233. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  8234. })
  8235. if err != nil {
  8236. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  8237. return
  8238. }
  8239. // c.TraceOn(os.Stderr)
  8240. testUserMetadataCopyingWrapper(c)
  8241. }
  8242. func testUserMetadataCopyingWrapper(c *minio.Client) {
  8243. // initialize logging params
  8244. startTime := time.Now()
  8245. testName := getFuncName()
  8246. function := "CopyObject(destination, source)"
  8247. args := map[string]interface{}{}
  8248. // Generate a new random bucket name.
  8249. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  8250. // Make a new bucket in 'us-east-1' (source bucket).
  8251. err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  8252. if err != nil {
  8253. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  8254. return
  8255. }
  8256. defer cleanupBucket(bucketName, c)
  8257. fetchMeta := func(object string) (h http.Header) {
  8258. objInfo, err := c.StatObject(context.Background(), bucketName, object, minio.StatObjectOptions{})
  8259. if err != nil {
  8260. logError(testName, function, args, startTime, "", "Stat failed", err)
  8261. return
  8262. }
  8263. h = make(http.Header)
  8264. for k, vs := range objInfo.Metadata {
  8265. if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") {
  8266. h.Add(k, vs[0])
  8267. }
  8268. }
  8269. return h
  8270. }
  8271. // 1. create a client encrypted object to copy by uploading
  8272. const srcSize = 1024 * 1024
  8273. buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB
  8274. metadata := make(http.Header)
  8275. metadata.Set("x-amz-meta-myheader", "myvalue")
  8276. m := make(map[string]string)
  8277. m["x-amz-meta-myheader"] = "myvalue"
  8278. _, err = c.PutObject(context.Background(), bucketName, "srcObject",
  8279. bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{UserMetadata: m})
  8280. if err != nil {
  8281. logError(testName, function, args, startTime, "", "PutObjectWithMetadata failed", err)
  8282. return
  8283. }
  8284. if !reflect.DeepEqual(metadata, fetchMeta("srcObject")) {
  8285. logError(testName, function, args, startTime, "", "Metadata match failed", err)
  8286. return
  8287. }
  8288. // 2. create source
  8289. src := minio.CopySrcOptions{
  8290. Bucket: bucketName,
  8291. Object: "srcObject",
  8292. }
  8293. // 2.1 create destination with metadata set
  8294. dst1 := minio.CopyDestOptions{
  8295. Bucket: bucketName,
  8296. Object: "dstObject-1",
  8297. UserMetadata: map[string]string{"notmyheader": "notmyvalue"},
  8298. ReplaceMetadata: true,
  8299. }
  8300. // 3. Check that copying to an object with metadata set resets
  8301. // the headers on the copy.
  8302. args["source"] = src
  8303. args["destination"] = dst1
  8304. _, err = c.CopyObject(context.Background(), dst1, src)
  8305. if err != nil {
  8306. logError(testName, function, args, startTime, "", "CopyObject failed", err)
  8307. return
  8308. }
  8309. expectedHeaders := make(http.Header)
  8310. expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue")
  8311. if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-1")) {
  8312. logError(testName, function, args, startTime, "", "Metadata match failed", err)
  8313. return
  8314. }
  8315. // 4. create destination with no metadata set and same source
  8316. dst2 := minio.CopyDestOptions{
  8317. Bucket: bucketName,
  8318. Object: "dstObject-2",
  8319. }
  8320. // 5. Check that copying to an object with no metadata set,
  8321. // copies metadata.
  8322. args["source"] = src
  8323. args["destination"] = dst2
  8324. _, err = c.CopyObject(context.Background(), dst2, src)
  8325. if err != nil {
  8326. logError(testName, function, args, startTime, "", "CopyObject failed", err)
  8327. return
  8328. }
  8329. expectedHeaders = metadata
  8330. if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-2")) {
  8331. logError(testName, function, args, startTime, "", "Metadata match failed", err)
  8332. return
  8333. }
  8334. // 6. Compose a pair of sources.
  8335. dst3 := minio.CopyDestOptions{
  8336. Bucket: bucketName,
  8337. Object: "dstObject-3",
  8338. ReplaceMetadata: true,
  8339. }
  8340. function = "ComposeObject(destination, sources)"
  8341. args["source"] = []minio.CopySrcOptions{src, src}
  8342. args["destination"] = dst3
  8343. _, err = c.ComposeObject(context.Background(), dst3, src, src)
  8344. if err != nil {
  8345. logError(testName, function, args, startTime, "", "ComposeObject failed", err)
  8346. return
  8347. }
  8348. // Check that no headers are copied in this case
  8349. if !reflect.DeepEqual(make(http.Header), fetchMeta("dstObject-3")) {
  8350. logError(testName, function, args, startTime, "", "Metadata match failed", err)
  8351. return
  8352. }
  8353. // 7. Compose a pair of sources with dest user metadata set.
  8354. dst4 := minio.CopyDestOptions{
  8355. Bucket: bucketName,
  8356. Object: "dstObject-4",
  8357. UserMetadata: map[string]string{"notmyheader": "notmyvalue"},
  8358. ReplaceMetadata: true,
  8359. }
  8360. function = "ComposeObject(destination, sources)"
  8361. args["source"] = []minio.CopySrcOptions{src, src}
  8362. args["destination"] = dst4
  8363. _, err = c.ComposeObject(context.Background(), dst4, src, src)
  8364. if err != nil {
  8365. logError(testName, function, args, startTime, "", "ComposeObject failed", err)
  8366. return
  8367. }
  8368. // Check that no headers are copied in this case
  8369. expectedHeaders = make(http.Header)
  8370. expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue")
  8371. if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-4")) {
  8372. logError(testName, function, args, startTime, "", "Metadata match failed", err)
  8373. return
  8374. }
  8375. successLogger(testName, function, args, startTime).Info()
  8376. }
  8377. func testUserMetadataCopyingV2() {
  8378. // initialize logging params
  8379. startTime := time.Now()
  8380. testName := getFuncName()
  8381. function := "CopyObject(destination, source)"
  8382. args := map[string]interface{}{}
  8383. // Instantiate new minio client object
  8384. c, err := minio.New(os.Getenv(serverEndpoint),
  8385. &minio.Options{
  8386. Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  8387. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  8388. })
  8389. if err != nil {
  8390. logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
  8391. return
  8392. }
  8393. // c.TraceOn(os.Stderr)
  8394. testUserMetadataCopyingWrapper(c)
  8395. }
  8396. func testStorageClassMetadataPutObject() {
  8397. // initialize logging params
  8398. startTime := time.Now()
  8399. function := "testStorageClassMetadataPutObject()"
  8400. args := map[string]interface{}{}
  8401. testName := getFuncName()
  8402. // Instantiate new minio client object
  8403. c, err := minio.New(os.Getenv(serverEndpoint),
  8404. &minio.Options{
  8405. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  8406. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  8407. })
  8408. if err != nil {
  8409. logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
  8410. return
  8411. }
  8412. // Generate a new random bucket name.
  8413. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
  8414. // Make a new bucket in 'us-east-1' (source bucket).
  8415. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  8416. if err != nil {
  8417. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  8418. return
  8419. }
  8420. defer cleanupBucket(bucketName, c)
  8421. fetchMeta := func(object string) (h http.Header) {
  8422. objInfo, err := c.StatObject(context.Background(), bucketName, object, minio.StatObjectOptions{})
  8423. if err != nil {
  8424. logError(testName, function, args, startTime, "", "Stat failed", err)
  8425. return
  8426. }
  8427. h = make(http.Header)
  8428. for k, vs := range objInfo.Metadata {
  8429. if strings.HasPrefix(strings.ToLower(k), "x-amz-storage-class") {
  8430. for _, v := range vs {
  8431. h.Add(k, v)
  8432. }
  8433. }
  8434. }
  8435. return h
  8436. }
  8437. metadata := make(http.Header)
  8438. metadata.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
  8439. emptyMetadata := make(http.Header)
  8440. const srcSize = 1024 * 1024
  8441. buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 1MiB
  8442. _, err = c.PutObject(context.Background(), bucketName, "srcObjectRRSClass",
  8443. bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "REDUCED_REDUNDANCY"})
  8444. if err != nil {
  8445. logError(testName, function, args, startTime, "", "PutObject failed", err)
  8446. return
  8447. }
  8448. // Get the returned metadata
  8449. returnedMeta := fetchMeta("srcObjectRRSClass")
  8450. // The response metada should either be equal to metadata (with REDUCED_REDUNDANCY) or emptyMetadata (in case of gateways)
  8451. if !reflect.DeepEqual(metadata, returnedMeta) && !reflect.DeepEqual(emptyMetadata, returnedMeta) {
  8452. logError(testName, function, args, startTime, "", "Metadata match failed", err)
  8453. return
  8454. }
  8455. metadata = make(http.Header)
  8456. metadata.Set("x-amz-storage-class", "STANDARD")
  8457. _, err = c.PutObject(context.Background(), bucketName, "srcObjectSSClass",
  8458. bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "STANDARD"})
  8459. if err != nil {
  8460. logError(testName, function, args, startTime, "", "PutObject failed", err)
  8461. return
  8462. }
  8463. if reflect.DeepEqual(metadata, fetchMeta("srcObjectSSClass")) {
  8464. logError(testName, function, args, startTime, "", "Metadata verification failed, STANDARD storage class should not be a part of response metadata", err)
  8465. return
  8466. }
  8467. successLogger(testName, function, args, startTime).Info()
  8468. }
  8469. func testStorageClassInvalidMetadataPutObject() {
  8470. // initialize logging params
  8471. startTime := time.Now()
  8472. function := "testStorageClassInvalidMetadataPutObject()"
  8473. args := map[string]interface{}{}
  8474. testName := getFuncName()
  8475. // Instantiate new minio client object
  8476. c, err := minio.New(os.Getenv(serverEndpoint),
  8477. &minio.Options{
  8478. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  8479. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  8480. })
  8481. if err != nil {
  8482. logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
  8483. return
  8484. }
  8485. // Generate a new random bucket name.
  8486. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
  8487. // Make a new bucket in 'us-east-1' (source bucket).
  8488. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  8489. if err != nil {
  8490. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  8491. return
  8492. }
  8493. defer cleanupBucket(bucketName, c)
  8494. const srcSize = 1024 * 1024
  8495. buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 1MiB
  8496. _, err = c.PutObject(context.Background(), bucketName, "srcObjectRRSClass",
  8497. bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "INVALID_STORAGE_CLASS"})
  8498. if err == nil {
  8499. logError(testName, function, args, startTime, "", "PutObject with invalid storage class passed, was expected to fail", err)
  8500. return
  8501. }
  8502. successLogger(testName, function, args, startTime).Info()
  8503. }
  8504. func testStorageClassMetadataCopyObject() {
  8505. // initialize logging params
  8506. startTime := time.Now()
  8507. function := "testStorageClassMetadataCopyObject()"
  8508. args := map[string]interface{}{}
  8509. testName := getFuncName()
  8510. // Instantiate new minio client object
  8511. c, err := minio.New(os.Getenv(serverEndpoint),
  8512. &minio.Options{
  8513. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  8514. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  8515. })
  8516. if err != nil {
  8517. logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
  8518. return
  8519. }
  8520. // Generate a new random bucket name.
  8521. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
  8522. // Make a new bucket in 'us-east-1' (source bucket).
  8523. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  8524. if err != nil {
  8525. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  8526. return
  8527. }
  8528. defer cleanupBucket(bucketName, c)
  8529. fetchMeta := func(object string) (h http.Header) {
  8530. objInfo, err := c.StatObject(context.Background(), bucketName, object, minio.StatObjectOptions{})
  8531. args["bucket"] = bucketName
  8532. args["object"] = object
  8533. if err != nil {
  8534. logError(testName, function, args, startTime, "", "Stat failed", err)
  8535. return
  8536. }
  8537. h = make(http.Header)
  8538. for k, vs := range objInfo.Metadata {
  8539. if strings.HasPrefix(strings.ToLower(k), "x-amz-storage-class") {
  8540. for _, v := range vs {
  8541. h.Add(k, v)
  8542. }
  8543. }
  8544. }
  8545. return h
  8546. }
  8547. metadata := make(http.Header)
  8548. metadata.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
  8549. emptyMetadata := make(http.Header)
  8550. const srcSize = 1024 * 1024
  8551. buf := bytes.Repeat([]byte("abcde"), srcSize)
  8552. // Put an object with RRS Storage class
  8553. _, err = c.PutObject(context.Background(), bucketName, "srcObjectRRSClass",
  8554. bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "REDUCED_REDUNDANCY"})
  8555. if err != nil {
  8556. logError(testName, function, args, startTime, "", "PutObject failed", err)
  8557. return
  8558. }
  8559. // Make server side copy of object uploaded in previous step
  8560. src := minio.CopySrcOptions{
  8561. Bucket: bucketName,
  8562. Object: "srcObjectRRSClass",
  8563. }
  8564. dst := minio.CopyDestOptions{
  8565. Bucket: bucketName,
  8566. Object: "srcObjectRRSClassCopy",
  8567. }
  8568. if _, err = c.CopyObject(context.Background(), dst, src); err != nil {
  8569. logError(testName, function, args, startTime, "", "CopyObject failed on RRS", err)
  8570. return
  8571. }
  8572. // Get the returned metadata
  8573. returnedMeta := fetchMeta("srcObjectRRSClassCopy")
  8574. // The response metada should either be equal to metadata (with REDUCED_REDUNDANCY) or emptyMetadata (in case of gateways)
  8575. if !reflect.DeepEqual(metadata, returnedMeta) && !reflect.DeepEqual(emptyMetadata, returnedMeta) {
  8576. logError(testName, function, args, startTime, "", "Metadata match failed", err)
  8577. return
  8578. }
  8579. metadata = make(http.Header)
  8580. metadata.Set("x-amz-storage-class", "STANDARD")
  8581. // Put an object with Standard Storage class
  8582. _, err = c.PutObject(context.Background(), bucketName, "srcObjectSSClass",
  8583. bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "STANDARD"})
  8584. if err != nil {
  8585. logError(testName, function, args, startTime, "", "PutObject failed", err)
  8586. return
  8587. }
  8588. // Make server side copy of object uploaded in previous step
  8589. src = minio.CopySrcOptions{
  8590. Bucket: bucketName,
  8591. Object: "srcObjectSSClass",
  8592. }
  8593. dst = minio.CopyDestOptions{
  8594. Bucket: bucketName,
  8595. Object: "srcObjectSSClassCopy",
  8596. }
  8597. if _, err = c.CopyObject(context.Background(), dst, src); err != nil {
  8598. logError(testName, function, args, startTime, "", "CopyObject failed on SS", err)
  8599. return
  8600. }
  8601. // Fetch the meta data of copied object
  8602. if reflect.DeepEqual(metadata, fetchMeta("srcObjectSSClassCopy")) {
  8603. logError(testName, function, args, startTime, "", "Metadata verification failed, STANDARD storage class should not be a part of response metadata", err)
  8604. return
  8605. }
  8606. successLogger(testName, function, args, startTime).Info()
  8607. }
  8608. // Test put object with size -1 byte object.
  8609. func testPutObjectNoLengthV2() {
  8610. // initialize logging params
  8611. startTime := time.Now()
  8612. testName := getFuncName()
  8613. function := "PutObject(bucketName, objectName, reader, size, opts)"
  8614. args := map[string]interface{}{
  8615. "bucketName": "",
  8616. "objectName": "",
  8617. "size": -1,
  8618. "opts": "",
  8619. }
  8620. // Seed random based on current time.
  8621. rand.Seed(time.Now().Unix())
  8622. // Instantiate new minio client object.
  8623. c, err := minio.New(os.Getenv(serverEndpoint),
  8624. &minio.Options{
  8625. Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  8626. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  8627. })
  8628. if err != nil {
  8629. logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
  8630. return
  8631. }
  8632. // Enable tracing, write to stderr.
  8633. // c.TraceOn(os.Stderr)
  8634. // Set user agent.
  8635. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  8636. // Generate a new random bucket name.
  8637. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  8638. args["bucketName"] = bucketName
  8639. // Make a new bucket.
  8640. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  8641. if err != nil {
  8642. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  8643. return
  8644. }
  8645. defer cleanupBucket(bucketName, c)
  8646. objectName := bucketName + "unique"
  8647. args["objectName"] = objectName
  8648. bufSize := dataFileMap["datafile-129-MB"]
  8649. var reader = getDataReader("datafile-129-MB")
  8650. defer reader.Close()
  8651. args["size"] = bufSize
  8652. // Upload an object.
  8653. _, err = c.PutObject(context.Background(), bucketName, objectName, reader, -1, minio.PutObjectOptions{})
  8654. if err != nil {
  8655. logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err)
  8656. return
  8657. }
  8658. st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
  8659. if err != nil {
  8660. logError(testName, function, args, startTime, "", "StatObject failed", err)
  8661. return
  8662. }
  8663. if st.Size != int64(bufSize) {
  8664. logError(testName, function, args, startTime, "", "Expected upload object size "+string(bufSize)+" got "+string(st.Size), err)
  8665. return
  8666. }
  8667. successLogger(testName, function, args, startTime).Info()
  8668. }
  8669. // Test put objects of unknown size.
  8670. func testPutObjectsUnknownV2() {
  8671. // initialize logging params
  8672. startTime := time.Now()
  8673. testName := getFuncName()
  8674. function := "PutObject(bucketName, objectName, reader,size,opts)"
  8675. args := map[string]interface{}{
  8676. "bucketName": "",
  8677. "objectName": "",
  8678. "size": "",
  8679. "opts": "",
  8680. }
  8681. // Seed random based on current time.
  8682. rand.Seed(time.Now().Unix())
  8683. // Instantiate new minio client object.
  8684. c, err := minio.New(os.Getenv(serverEndpoint),
  8685. &minio.Options{
  8686. Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  8687. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  8688. })
  8689. if err != nil {
  8690. logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
  8691. return
  8692. }
  8693. // Enable tracing, write to stderr.
  8694. // c.TraceOn(os.Stderr)
  8695. // Set user agent.
  8696. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  8697. // Generate a new random bucket name.
  8698. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  8699. args["bucketName"] = bucketName
  8700. // Make a new bucket.
  8701. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  8702. if err != nil {
  8703. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  8704. return
  8705. }
  8706. defer cleanupBucket(bucketName, c)
  8707. // Issues are revealed by trying to upload multiple files of unknown size
  8708. // sequentially (on 4GB machines)
  8709. for i := 1; i <= 4; i++ {
  8710. // Simulate that we could be receiving byte slices of data that we want
  8711. // to upload as a file
  8712. rpipe, wpipe := io.Pipe()
  8713. defer rpipe.Close()
  8714. go func() {
  8715. b := []byte("test")
  8716. wpipe.Write(b)
  8717. wpipe.Close()
  8718. }()
  8719. // Upload the object.
  8720. objectName := fmt.Sprintf("%sunique%d", bucketName, i)
  8721. args["objectName"] = objectName
  8722. ui, err := c.PutObject(context.Background(), bucketName, objectName, rpipe, -1, minio.PutObjectOptions{})
  8723. if err != nil {
  8724. logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err)
  8725. return
  8726. }
  8727. if ui.Size != 4 {
  8728. logError(testName, function, args, startTime, "", "Expected upload object size "+string(4)+" got "+string(ui.Size), nil)
  8729. return
  8730. }
  8731. st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
  8732. if err != nil {
  8733. logError(testName, function, args, startTime, "", "StatObjectStreaming failed", err)
  8734. return
  8735. }
  8736. if st.Size != int64(4) {
  8737. logError(testName, function, args, startTime, "", "Expected upload object size "+string(4)+" got "+string(st.Size), err)
  8738. return
  8739. }
  8740. }
  8741. successLogger(testName, function, args, startTime).Info()
  8742. }
  8743. // Test put object with 0 byte object.
  8744. func testPutObject0ByteV2() {
  8745. // initialize logging params
  8746. startTime := time.Now()
  8747. testName := getFuncName()
  8748. function := "PutObject(bucketName, objectName, reader, size, opts)"
  8749. args := map[string]interface{}{
  8750. "bucketName": "",
  8751. "objectName": "",
  8752. "size": 0,
  8753. "opts": "",
  8754. }
  8755. // Seed random based on current time.
  8756. rand.Seed(time.Now().Unix())
  8757. // Instantiate new minio client object.
  8758. c, err := minio.New(os.Getenv(serverEndpoint),
  8759. &minio.Options{
  8760. Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  8761. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  8762. })
  8763. if err != nil {
  8764. logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
  8765. return
  8766. }
  8767. // Enable tracing, write to stderr.
  8768. // c.TraceOn(os.Stderr)
  8769. // Set user agent.
  8770. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  8771. // Generate a new random bucket name.
  8772. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  8773. args["bucketName"] = bucketName
  8774. // Make a new bucket.
  8775. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  8776. if err != nil {
  8777. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  8778. return
  8779. }
  8780. defer cleanupBucket(bucketName, c)
  8781. objectName := bucketName + "unique"
  8782. args["objectName"] = objectName
  8783. args["opts"] = minio.PutObjectOptions{}
  8784. // Upload an object.
  8785. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader([]byte("")), 0, minio.PutObjectOptions{})
  8786. if err != nil {
  8787. logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err)
  8788. return
  8789. }
  8790. st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
  8791. if err != nil {
  8792. logError(testName, function, args, startTime, "", "StatObjectWithSize failed", err)
  8793. return
  8794. }
  8795. if st.Size != 0 {
  8796. logError(testName, function, args, startTime, "", "Expected upload object size 0 but got "+string(st.Size), err)
  8797. return
  8798. }
  8799. successLogger(testName, function, args, startTime).Info()
  8800. }
  8801. // Test expected error cases
  8802. func testComposeObjectErrorCases() {
  8803. // initialize logging params
  8804. startTime := time.Now()
  8805. testName := getFuncName()
  8806. function := "ComposeObject(destination, sourceList)"
  8807. args := map[string]interface{}{}
  8808. // Instantiate new minio client object
  8809. c, err := minio.New(os.Getenv(serverEndpoint),
  8810. &minio.Options{
  8811. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  8812. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  8813. })
  8814. if err != nil {
  8815. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  8816. return
  8817. }
  8818. testComposeObjectErrorCasesWrapper(c)
  8819. }
  8820. // Test concatenating multiple 10K objects V4
  8821. func testCompose10KSources() {
  8822. // initialize logging params
  8823. startTime := time.Now()
  8824. testName := getFuncName()
  8825. function := "ComposeObject(destination, sourceList)"
  8826. args := map[string]interface{}{}
  8827. // Instantiate new minio client object
  8828. c, err := minio.New(os.Getenv(serverEndpoint),
  8829. &minio.Options{
  8830. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  8831. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  8832. })
  8833. if err != nil {
  8834. logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
  8835. return
  8836. }
  8837. testComposeMultipleSources(c)
  8838. }
  8839. // Tests comprehensive list of all methods.
  8840. func testFunctionalV2() {
  8841. // initialize logging params
  8842. startTime := time.Now()
  8843. testName := getFuncName()
  8844. function := "testFunctionalV2()"
  8845. functionAll := ""
  8846. args := map[string]interface{}{}
  8847. // Seed random based on current time.
  8848. rand.Seed(time.Now().Unix())
  8849. c, err := minio.New(os.Getenv(serverEndpoint),
  8850. &minio.Options{
  8851. Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  8852. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  8853. })
  8854. if err != nil {
  8855. logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
  8856. return
  8857. }
  8858. // Enable to debug
  8859. // c.TraceOn(os.Stderr)
  8860. // Set user agent.
  8861. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  8862. // Generate a new random bucket name.
  8863. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  8864. location := "us-east-1"
  8865. // Make a new bucket.
  8866. function = "MakeBucket(bucketName, location)"
  8867. functionAll = "MakeBucket(bucketName, location)"
  8868. args = map[string]interface{}{
  8869. "bucketName": bucketName,
  8870. "location": location,
  8871. }
  8872. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: location})
  8873. if err != nil {
  8874. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  8875. return
  8876. }
  8877. defer cleanupBucket(bucketName, c)
  8878. // Generate a random file name.
  8879. fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  8880. file, err := os.Create(fileName)
  8881. if err != nil {
  8882. logError(testName, function, args, startTime, "", "file create failed", err)
  8883. return
  8884. }
  8885. for i := 0; i < 3; i++ {
  8886. buf := make([]byte, rand.Intn(1<<19))
  8887. _, err = file.Write(buf)
  8888. if err != nil {
  8889. logError(testName, function, args, startTime, "", "file write failed", err)
  8890. return
  8891. }
  8892. }
  8893. file.Close()
  8894. // Verify if bucket exits and you have access.
  8895. var exists bool
  8896. function = "BucketExists(bucketName)"
  8897. functionAll += ", " + function
  8898. args = map[string]interface{}{
  8899. "bucketName": bucketName,
  8900. }
  8901. exists, err = c.BucketExists(context.Background(), bucketName)
  8902. if err != nil {
  8903. logError(testName, function, args, startTime, "", "BucketExists failed", err)
  8904. return
  8905. }
  8906. if !exists {
  8907. logError(testName, function, args, startTime, "", "Could not find existing bucket "+bucketName, err)
  8908. return
  8909. }
  8910. // Make the bucket 'public read/write'.
  8911. function = "SetBucketPolicy(bucketName, bucketPolicy)"
  8912. functionAll += ", " + function
  8913. readWritePolicy := `{"Version": "2012-10-17","Statement": [{"Action": ["s3:ListBucketMultipartUploads", "s3:ListBucket"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::` + bucketName + `"],"Sid": ""}]}`
  8914. args = map[string]interface{}{
  8915. "bucketName": bucketName,
  8916. "bucketPolicy": readWritePolicy,
  8917. }
  8918. err = c.SetBucketPolicy(context.Background(), bucketName, readWritePolicy)
  8919. if err != nil {
  8920. logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err)
  8921. return
  8922. }
  8923. // List all buckets.
  8924. function = "ListBuckets()"
  8925. functionAll += ", " + function
  8926. args = nil
  8927. buckets, err := c.ListBuckets(context.Background())
  8928. if len(buckets) == 0 {
  8929. logError(testName, function, args, startTime, "", "List buckets cannot be empty", err)
  8930. return
  8931. }
  8932. if err != nil {
  8933. logError(testName, function, args, startTime, "", "ListBuckets failed", err)
  8934. return
  8935. }
  8936. // Verify if previously created bucket is listed in list buckets.
  8937. bucketFound := false
  8938. for _, bucket := range buckets {
  8939. if bucket.Name == bucketName {
  8940. bucketFound = true
  8941. }
  8942. }
  8943. // If bucket not found error out.
  8944. if !bucketFound {
  8945. logError(testName, function, args, startTime, "", "Bucket "+bucketName+"not found", err)
  8946. return
  8947. }
  8948. objectName := bucketName + "unique"
  8949. // Generate data
  8950. buf := bytes.Repeat([]byte("n"), rand.Intn(1<<19))
  8951. args = map[string]interface{}{
  8952. "bucketName": bucketName,
  8953. "objectName": objectName,
  8954. "contentType": "",
  8955. }
  8956. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{})
  8957. if err != nil {
  8958. logError(testName, function, args, startTime, "", "PutObject failed", err)
  8959. return
  8960. }
  8961. st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
  8962. if err != nil {
  8963. logError(testName, function, args, startTime, "", "StatObject failed", err)
  8964. return
  8965. }
  8966. if st.Size != int64(len(buf)) {
  8967. logError(testName, function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(st.Size), err)
  8968. return
  8969. }
  8970. objectNameNoLength := objectName + "-nolength"
  8971. args["objectName"] = objectNameNoLength
  8972. _, err = c.PutObject(context.Background(), bucketName, objectNameNoLength, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  8973. if err != nil {
  8974. logError(testName, function, args, startTime, "", "PutObject failed", err)
  8975. return
  8976. }
  8977. st, err = c.StatObject(context.Background(), bucketName, objectNameNoLength, minio.StatObjectOptions{})
  8978. if err != nil {
  8979. logError(testName, function, args, startTime, "", "StatObject failed", err)
  8980. return
  8981. }
  8982. if st.Size != int64(len(buf)) {
  8983. logError(testName, function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(st.Size), err)
  8984. return
  8985. }
  8986. // Instantiate a done channel to close all listing.
  8987. doneCh := make(chan struct{})
  8988. defer close(doneCh)
  8989. objFound := false
  8990. isRecursive := true // Recursive is true.
  8991. function = "ListObjects(bucketName, objectName, isRecursive, doneCh)"
  8992. functionAll += ", " + function
  8993. args = map[string]interface{}{
  8994. "bucketName": bucketName,
  8995. "objectName": objectName,
  8996. "isRecursive": isRecursive,
  8997. }
  8998. for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Prefix: objectName, Recursive: isRecursive}) {
  8999. if obj.Key == objectName {
  9000. objFound = true
  9001. break
  9002. }
  9003. }
  9004. if !objFound {
  9005. logError(testName, function, args, startTime, "", "Could not find existing object "+objectName, err)
  9006. return
  9007. }
  9008. incompObjNotFound := true
  9009. function = "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)"
  9010. functionAll += ", " + function
  9011. args = map[string]interface{}{
  9012. "bucketName": bucketName,
  9013. "objectName": objectName,
  9014. "isRecursive": isRecursive,
  9015. }
  9016. for objIncompl := range c.ListIncompleteUploads(context.Background(), bucketName, objectName, isRecursive) {
  9017. if objIncompl.Key != "" {
  9018. incompObjNotFound = false
  9019. break
  9020. }
  9021. }
  9022. if !incompObjNotFound {
  9023. logError(testName, function, args, startTime, "", "Unexpected dangling incomplete upload found", err)
  9024. return
  9025. }
  9026. function = "GetObject(bucketName, objectName)"
  9027. functionAll += ", " + function
  9028. args = map[string]interface{}{
  9029. "bucketName": bucketName,
  9030. "objectName": objectName,
  9031. }
  9032. newReader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
  9033. if err != nil {
  9034. logError(testName, function, args, startTime, "", "GetObject failed", err)
  9035. return
  9036. }
  9037. newReadBytes, err := ioutil.ReadAll(newReader)
  9038. if err != nil {
  9039. logError(testName, function, args, startTime, "", "ReadAll failed", err)
  9040. return
  9041. }
  9042. newReader.Close()
  9043. if !bytes.Equal(newReadBytes, buf) {
  9044. logError(testName, function, args, startTime, "", "Bytes mismatch", err)
  9045. return
  9046. }
  9047. function = "FGetObject(bucketName, objectName, fileName)"
  9048. functionAll += ", " + function
  9049. args = map[string]interface{}{
  9050. "bucketName": bucketName,
  9051. "objectName": objectName,
  9052. "fileName": fileName + "-f",
  9053. }
  9054. err = c.FGetObject(context.Background(), bucketName, objectName, fileName+"-f", minio.GetObjectOptions{})
  9055. if err != nil {
  9056. logError(testName, function, args, startTime, "", "FgetObject failed", err)
  9057. return
  9058. }
  9059. // Generate presigned HEAD object url.
  9060. function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)"
  9061. functionAll += ", " + function
  9062. args = map[string]interface{}{
  9063. "bucketName": bucketName,
  9064. "objectName": objectName,
  9065. "expires": 3600 * time.Second,
  9066. }
  9067. presignedHeadURL, err := c.PresignedHeadObject(context.Background(), bucketName, objectName, 3600*time.Second, nil)
  9068. if err != nil {
  9069. logError(testName, function, args, startTime, "", "PresignedHeadObject failed", err)
  9070. return
  9071. }
  9072. transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS)))
  9073. if err != nil {
  9074. logError(testName, function, args, startTime, "", "DefaultTransport failed", err)
  9075. return
  9076. }
  9077. httpClient := &http.Client{
  9078. // Setting a sensible time out of 30secs to wait for response
  9079. // headers. Request is pro-actively canceled after 30secs
  9080. // with no response.
  9081. Timeout: 30 * time.Second,
  9082. Transport: transport,
  9083. }
  9084. req, err := http.NewRequest(http.MethodHead, presignedHeadURL.String(), nil)
  9085. if err != nil {
  9086. logError(testName, function, args, startTime, "", "PresignedHeadObject URL head request failed", err)
  9087. return
  9088. }
  9089. // Verify if presigned url works.
  9090. resp, err := httpClient.Do(req)
  9091. if err != nil {
  9092. logError(testName, function, args, startTime, "", "PresignedHeadObject URL head request failed", err)
  9093. return
  9094. }
  9095. if resp.StatusCode != http.StatusOK {
  9096. logError(testName, function, args, startTime, "", "PresignedHeadObject URL returns status "+string(resp.StatusCode), err)
  9097. return
  9098. }
  9099. if resp.Header.Get("ETag") == "" {
  9100. logError(testName, function, args, startTime, "", "Got empty ETag", err)
  9101. return
  9102. }
  9103. resp.Body.Close()
  9104. // Generate presigned GET object url.
  9105. function = "PresignedGetObject(bucketName, objectName, expires, reqParams)"
  9106. functionAll += ", " + function
  9107. args = map[string]interface{}{
  9108. "bucketName": bucketName,
  9109. "objectName": objectName,
  9110. "expires": 3600 * time.Second,
  9111. }
  9112. presignedGetURL, err := c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, nil)
  9113. if err != nil {
  9114. logError(testName, function, args, startTime, "", "PresignedGetObject failed", err)
  9115. return
  9116. }
  9117. // Verify if presigned url works.
  9118. req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil)
  9119. if err != nil {
  9120. logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err)
  9121. return
  9122. }
  9123. resp, err = httpClient.Do(req)
  9124. if err != nil {
  9125. logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
  9126. return
  9127. }
  9128. if resp.StatusCode != http.StatusOK {
  9129. logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err)
  9130. return
  9131. }
  9132. newPresignedBytes, err := ioutil.ReadAll(resp.Body)
  9133. if err != nil {
  9134. logError(testName, function, args, startTime, "", "ReadAll failed", err)
  9135. return
  9136. }
  9137. resp.Body.Close()
  9138. if !bytes.Equal(newPresignedBytes, buf) {
  9139. logError(testName, function, args, startTime, "", "Bytes mismatch", err)
  9140. return
  9141. }
  9142. // Set request parameters.
  9143. reqParams := make(url.Values)
  9144. reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"")
  9145. // Generate presigned GET object url.
  9146. args["reqParams"] = reqParams
  9147. presignedGetURL, err = c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, reqParams)
  9148. if err != nil {
  9149. logError(testName, function, args, startTime, "", "PresignedGetObject failed", err)
  9150. return
  9151. }
  9152. // Verify if presigned url works.
  9153. req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil)
  9154. if err != nil {
  9155. logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err)
  9156. return
  9157. }
  9158. resp, err = httpClient.Do(req)
  9159. if err != nil {
  9160. logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
  9161. return
  9162. }
  9163. if resp.StatusCode != http.StatusOK {
  9164. logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err)
  9165. return
  9166. }
  9167. newPresignedBytes, err = ioutil.ReadAll(resp.Body)
  9168. if err != nil {
  9169. logError(testName, function, args, startTime, "", "ReadAll failed", err)
  9170. return
  9171. }
  9172. if !bytes.Equal(newPresignedBytes, buf) {
  9173. logError(testName, function, args, startTime, "", "Bytes mismatch", err)
  9174. return
  9175. }
  9176. // Verify content disposition.
  9177. if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" {
  9178. logError(testName, function, args, startTime, "", "wrong Content-Disposition received ", err)
  9179. return
  9180. }
  9181. function = "PresignedPutObject(bucketName, objectName, expires)"
  9182. functionAll += ", " + function
  9183. args = map[string]interface{}{
  9184. "bucketName": bucketName,
  9185. "objectName": objectName + "-presigned",
  9186. "expires": 3600 * time.Second,
  9187. }
  9188. presignedPutURL, err := c.PresignedPutObject(context.Background(), bucketName, objectName+"-presigned", 3600*time.Second)
  9189. if err != nil {
  9190. logError(testName, function, args, startTime, "", "PresignedPutObject failed", err)
  9191. return
  9192. }
  9193. // Generate data more than 32K
  9194. buf = bytes.Repeat([]byte("1"), rand.Intn(1<<10)+32*1024)
  9195. req, err = http.NewRequest(http.MethodPut, presignedPutURL.String(), bytes.NewReader(buf))
  9196. if err != nil {
  9197. logError(testName, function, args, startTime, "", "HTTP request to PresignedPutObject URL failed", err)
  9198. return
  9199. }
  9200. resp, err = httpClient.Do(req)
  9201. if err != nil {
  9202. logError(testName, function, args, startTime, "", "HTTP request to PresignedPutObject URL failed", err)
  9203. return
  9204. }
  9205. function = "GetObject(bucketName, objectName)"
  9206. functionAll += ", " + function
  9207. args = map[string]interface{}{
  9208. "bucketName": bucketName,
  9209. "objectName": objectName + "-presigned",
  9210. }
  9211. newReader, err = c.GetObject(context.Background(), bucketName, objectName+"-presigned", minio.GetObjectOptions{})
  9212. if err != nil {
  9213. logError(testName, function, args, startTime, "", "GetObject failed", err)
  9214. return
  9215. }
  9216. newReadBytes, err = ioutil.ReadAll(newReader)
  9217. if err != nil {
  9218. logError(testName, function, args, startTime, "", "ReadAll failed", err)
  9219. return
  9220. }
  9221. newReader.Close()
  9222. if !bytes.Equal(newReadBytes, buf) {
  9223. logError(testName, function, args, startTime, "", "Bytes mismatch", err)
  9224. return
  9225. }
  9226. os.Remove(fileName)
  9227. os.Remove(fileName + "-f")
  9228. successLogger(testName, functionAll, args, startTime).Info()
  9229. }
  9230. // Test get object with GetObject with context
  9231. func testGetObjectContext() {
  9232. // initialize logging params
  9233. startTime := time.Now()
  9234. testName := getFuncName()
  9235. function := "GetObject(ctx, bucketName, objectName)"
  9236. args := map[string]interface{}{
  9237. "ctx": "",
  9238. "bucketName": "",
  9239. "objectName": "",
  9240. }
  9241. // Seed random based on current time.
  9242. rand.Seed(time.Now().Unix())
  9243. // Instantiate new minio client object.
  9244. c, err := minio.New(os.Getenv(serverEndpoint),
  9245. &minio.Options{
  9246. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  9247. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  9248. })
  9249. if err != nil {
  9250. logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err)
  9251. return
  9252. }
  9253. // Enable tracing, write to stderr.
  9254. // c.TraceOn(os.Stderr)
  9255. // Set user agent.
  9256. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  9257. // Generate a new random bucket name.
  9258. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  9259. args["bucketName"] = bucketName
  9260. // Make a new bucket.
  9261. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  9262. if err != nil {
  9263. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  9264. return
  9265. }
  9266. defer cleanupBucket(bucketName, c)
  9267. bufSize := dataFileMap["datafile-33-kB"]
  9268. var reader = getDataReader("datafile-33-kB")
  9269. defer reader.Close()
  9270. // Save the data
  9271. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  9272. args["objectName"] = objectName
  9273. _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  9274. if err != nil {
  9275. logError(testName, function, args, startTime, "", "PutObject failed", err)
  9276. return
  9277. }
  9278. ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
  9279. args["ctx"] = ctx
  9280. cancel()
  9281. r, err := c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{})
  9282. if err != nil {
  9283. logError(testName, function, args, startTime, "", "GetObject failed unexpectedly", err)
  9284. return
  9285. }
  9286. if _, err = r.Stat(); err == nil {
  9287. logError(testName, function, args, startTime, "", "GetObject should fail on short timeout", err)
  9288. return
  9289. }
  9290. r.Close()
  9291. ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
  9292. args["ctx"] = ctx
  9293. defer cancel()
  9294. // Read the data back
  9295. r, err = c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{})
  9296. if err != nil {
  9297. logError(testName, function, args, startTime, "", "GetObject failed", err)
  9298. return
  9299. }
  9300. st, err := r.Stat()
  9301. if err != nil {
  9302. logError(testName, function, args, startTime, "", "object Stat call failed", err)
  9303. return
  9304. }
  9305. if st.Size != int64(bufSize) {
  9306. logError(testName, function, args, startTime, "", "Number of bytes in stat does not match: want "+string(bufSize)+", got"+string(st.Size), err)
  9307. return
  9308. }
  9309. if err := r.Close(); err != nil {
  9310. logError(testName, function, args, startTime, "", "object Close() call failed", err)
  9311. return
  9312. }
  9313. successLogger(testName, function, args, startTime).Info()
  9314. }
  9315. // Test get object with FGetObject with a user provided context
  9316. func testFGetObjectContext() {
  9317. // initialize logging params
  9318. startTime := time.Now()
  9319. testName := getFuncName()
  9320. function := "FGetObject(ctx, bucketName, objectName, fileName)"
  9321. args := map[string]interface{}{
  9322. "ctx": "",
  9323. "bucketName": "",
  9324. "objectName": "",
  9325. "fileName": "",
  9326. }
  9327. // Seed random based on current time.
  9328. rand.Seed(time.Now().Unix())
  9329. // Instantiate new minio client object.
  9330. c, err := minio.New(os.Getenv(serverEndpoint),
  9331. &minio.Options{
  9332. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  9333. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  9334. })
  9335. if err != nil {
  9336. logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err)
  9337. return
  9338. }
  9339. // Enable tracing, write to stderr.
  9340. // c.TraceOn(os.Stderr)
  9341. // Set user agent.
  9342. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  9343. // Generate a new random bucket name.
  9344. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  9345. args["bucketName"] = bucketName
  9346. // Make a new bucket.
  9347. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  9348. if err != nil {
  9349. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  9350. return
  9351. }
  9352. defer cleanupBucket(bucketName, c)
  9353. bufSize := dataFileMap["datafile-1-MB"]
  9354. var reader = getDataReader("datafile-1-MB")
  9355. defer reader.Close()
  9356. // Save the data
  9357. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  9358. args["objectName"] = objectName
  9359. _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  9360. if err != nil {
  9361. logError(testName, function, args, startTime, "", "PutObject failed", err)
  9362. return
  9363. }
  9364. ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
  9365. args["ctx"] = ctx
  9366. defer cancel()
  9367. fileName := "tempfile-context"
  9368. args["fileName"] = fileName
  9369. // Read the data back
  9370. err = c.FGetObject(ctx, bucketName, objectName, fileName+"-f", minio.GetObjectOptions{})
  9371. if err == nil {
  9372. logError(testName, function, args, startTime, "", "FGetObject should fail on short timeout", err)
  9373. return
  9374. }
  9375. ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
  9376. defer cancel()
  9377. // Read the data back
  9378. err = c.FGetObject(ctx, bucketName, objectName, fileName+"-fcontext", minio.GetObjectOptions{})
  9379. if err != nil {
  9380. logError(testName, function, args, startTime, "", "FGetObject with long timeout failed", err)
  9381. return
  9382. }
  9383. if err = os.Remove(fileName + "-fcontext"); err != nil {
  9384. logError(testName, function, args, startTime, "", "Remove file failed", err)
  9385. return
  9386. }
  9387. successLogger(testName, function, args, startTime).Info()
  9388. }
  9389. // Test get object with GetObject with a user provided context
  9390. func testGetObjectRanges() {
  9391. // initialize logging params
  9392. startTime := time.Now()
  9393. testName := getFuncName()
  9394. function := "GetObject(ctx, bucketName, objectName, fileName)"
  9395. args := map[string]interface{}{
  9396. "ctx": "",
  9397. "bucketName": "",
  9398. "objectName": "",
  9399. "fileName": "",
  9400. }
  9401. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
  9402. defer cancel()
  9403. rng := rand.NewSource(time.Now().UnixNano())
  9404. // Instantiate new minio client object.
  9405. c, err := minio.New(os.Getenv(serverEndpoint),
  9406. &minio.Options{
  9407. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  9408. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  9409. })
  9410. if err != nil {
  9411. logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err)
  9412. return
  9413. }
  9414. // Enable tracing, write to stderr.
  9415. // c.TraceOn(os.Stderr)
  9416. // Set user agent.
  9417. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  9418. // Generate a new random bucket name.
  9419. bucketName := randString(60, rng, "minio-go-test-")
  9420. args["bucketName"] = bucketName
  9421. // Make a new bucket.
  9422. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  9423. if err != nil {
  9424. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  9425. return
  9426. }
  9427. defer cleanupBucket(bucketName, c)
  9428. bufSize := dataFileMap["datafile-129-MB"]
  9429. var reader = getDataReader("datafile-129-MB")
  9430. defer reader.Close()
  9431. // Save the data
  9432. objectName := randString(60, rng, "")
  9433. args["objectName"] = objectName
  9434. _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  9435. if err != nil {
  9436. logError(testName, function, args, startTime, "", "PutObject failed", err)
  9437. return
  9438. }
  9439. // Read the data back
  9440. tests := []struct {
  9441. start int64
  9442. end int64
  9443. }{
  9444. {
  9445. start: 1024,
  9446. end: 1024 + 1<<20,
  9447. },
  9448. {
  9449. start: 20e6,
  9450. end: 20e6 + 10000,
  9451. },
  9452. {
  9453. start: 40e6,
  9454. end: 40e6 + 10000,
  9455. },
  9456. {
  9457. start: 60e6,
  9458. end: 60e6 + 10000,
  9459. },
  9460. {
  9461. start: 80e6,
  9462. end: 80e6 + 10000,
  9463. },
  9464. {
  9465. start: 120e6,
  9466. end: int64(bufSize),
  9467. },
  9468. }
  9469. for _, test := range tests {
  9470. wantRC := getDataReader("datafile-129-MB")
  9471. io.CopyN(ioutil.Discard, wantRC, test.start)
  9472. want := mustCrcReader(io.LimitReader(wantRC, test.end-test.start+1))
  9473. opts := minio.GetObjectOptions{}
  9474. opts.SetRange(test.start, test.end)
  9475. args["opts"] = fmt.Sprintf("%+v", test)
  9476. obj, err := c.GetObject(ctx, bucketName, objectName, opts)
  9477. if err != nil {
  9478. logError(testName, function, args, startTime, "", "FGetObject with long timeout failed", err)
  9479. return
  9480. }
  9481. err = crcMatches(obj, want)
  9482. if err != nil {
  9483. logError(testName, function, args, startTime, "", fmt.Sprintf("GetObject offset %d -> %d", test.start, test.end), err)
  9484. return
  9485. }
  9486. }
  9487. successLogger(testName, function, args, startTime).Info()
  9488. }
  9489. // Test get object ACLs with GetObjectACL with custom provided context
  9490. func testGetObjectACLContext() {
  9491. // initialize logging params
  9492. startTime := time.Now()
  9493. testName := getFuncName()
  9494. function := "GetObjectACL(ctx, bucketName, objectName)"
  9495. args := map[string]interface{}{
  9496. "ctx": "",
  9497. "bucketName": "",
  9498. "objectName": "",
  9499. }
  9500. // Seed random based on current time.
  9501. rand.Seed(time.Now().Unix())
  9502. // skipping region functional tests for non s3 runs
  9503. if os.Getenv(serverEndpoint) != "s3.amazonaws.com" {
  9504. ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info()
  9505. return
  9506. }
  9507. // Instantiate new minio client object.
  9508. c, err := minio.New(os.Getenv(serverEndpoint),
  9509. &minio.Options{
  9510. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  9511. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  9512. })
  9513. if err != nil {
  9514. logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err)
  9515. return
  9516. }
  9517. // Enable tracing, write to stderr.
  9518. // c.TraceOn(os.Stderr)
  9519. // Set user agent.
  9520. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  9521. // Generate a new random bucket name.
  9522. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  9523. args["bucketName"] = bucketName
  9524. // Make a new bucket.
  9525. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  9526. if err != nil {
  9527. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  9528. return
  9529. }
  9530. defer cleanupBucket(bucketName, c)
  9531. bufSize := dataFileMap["datafile-1-MB"]
  9532. var reader = getDataReader("datafile-1-MB")
  9533. defer reader.Close()
  9534. // Save the data
  9535. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  9536. args["objectName"] = objectName
  9537. // Add meta data to add a canned acl
  9538. metaData := map[string]string{
  9539. "X-Amz-Acl": "public-read-write",
  9540. }
  9541. _, err = c.PutObject(context.Background(), bucketName,
  9542. objectName, reader, int64(bufSize),
  9543. minio.PutObjectOptions{
  9544. ContentType: "binary/octet-stream",
  9545. UserMetadata: metaData,
  9546. })
  9547. if err != nil {
  9548. logError(testName, function, args, startTime, "", "PutObject failed", err)
  9549. return
  9550. }
  9551. ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
  9552. args["ctx"] = ctx
  9553. defer cancel()
  9554. // Read the data back
  9555. objectInfo, getObjectACLErr := c.GetObjectACL(ctx, bucketName, objectName)
  9556. if getObjectACLErr != nil {
  9557. logError(testName, function, args, startTime, "", "GetObjectACL failed. ", getObjectACLErr)
  9558. return
  9559. }
  9560. s, ok := objectInfo.Metadata["X-Amz-Acl"]
  9561. if !ok {
  9562. logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Acl\"", nil)
  9563. return
  9564. }
  9565. if len(s) != 1 {
  9566. logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" canned acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil)
  9567. return
  9568. }
  9569. if s[0] != "public-read-write" {
  9570. logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" expected \"public-read-write\" but got"+fmt.Sprintf("%q", s[0]), nil)
  9571. return
  9572. }
  9573. bufSize = dataFileMap["datafile-1-MB"]
  9574. var reader2 = getDataReader("datafile-1-MB")
  9575. defer reader2.Close()
  9576. // Save the data
  9577. objectName = randString(60, rand.NewSource(time.Now().UnixNano()), "")
  9578. args["objectName"] = objectName
  9579. // Add meta data to add a canned acl
  9580. metaData = map[string]string{
  9581. "X-Amz-Grant-Read": "id=fooread@minio.go",
  9582. "X-Amz-Grant-Write": "id=foowrite@minio.go",
  9583. }
  9584. _, err = c.PutObject(context.Background(), bucketName, objectName, reader2, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", UserMetadata: metaData})
  9585. if err != nil {
  9586. logError(testName, function, args, startTime, "", "PutObject failed", err)
  9587. return
  9588. }
  9589. ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
  9590. args["ctx"] = ctx
  9591. defer cancel()
  9592. // Read the data back
  9593. objectInfo, getObjectACLErr = c.GetObjectACL(ctx, bucketName, objectName)
  9594. if getObjectACLErr == nil {
  9595. logError(testName, function, args, startTime, "", "GetObjectACL fail", getObjectACLErr)
  9596. return
  9597. }
  9598. if len(objectInfo.Metadata) != 3 {
  9599. logError(testName, function, args, startTime, "", "GetObjectACL fail expected \"3\" ACLs but got "+fmt.Sprintf(`"%d"`, len(objectInfo.Metadata)), nil)
  9600. return
  9601. }
  9602. s, ok = objectInfo.Metadata["X-Amz-Grant-Read"]
  9603. if !ok {
  9604. logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Grant-Read\"", nil)
  9605. return
  9606. }
  9607. if len(s) != 1 {
  9608. logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Read\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil)
  9609. return
  9610. }
  9611. if s[0] != "fooread@minio.go" {
  9612. logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Read\" acl expected \"fooread@minio.go\" got "+fmt.Sprintf("%q", s), nil)
  9613. return
  9614. }
  9615. s, ok = objectInfo.Metadata["X-Amz-Grant-Write"]
  9616. if !ok {
  9617. logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Grant-Write\"", nil)
  9618. return
  9619. }
  9620. if len(s) != 1 {
  9621. logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Write\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil)
  9622. return
  9623. }
  9624. if s[0] != "foowrite@minio.go" {
  9625. logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Write\" acl expected \"foowrite@minio.go\" got "+fmt.Sprintf("%q", s), nil)
  9626. return
  9627. }
  9628. successLogger(testName, function, args, startTime).Info()
  9629. }
  9630. // Test validates putObject with context to see if request cancellation is honored for V2.
  9631. func testPutObjectContextV2() {
  9632. // initialize logging params
  9633. startTime := time.Now()
  9634. testName := getFuncName()
  9635. function := "PutObject(ctx, bucketName, objectName, reader, size, opts)"
  9636. args := map[string]interface{}{
  9637. "ctx": "",
  9638. "bucketName": "",
  9639. "objectName": "",
  9640. "size": "",
  9641. "opts": "",
  9642. }
  9643. // Instantiate new minio client object.
  9644. c, err := minio.New(os.Getenv(serverEndpoint),
  9645. &minio.Options{
  9646. Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  9647. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  9648. })
  9649. if err != nil {
  9650. logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
  9651. return
  9652. }
  9653. // Enable tracing, write to stderr.
  9654. // c.TraceOn(os.Stderr)
  9655. // Set user agent.
  9656. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  9657. // Make a new bucket.
  9658. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  9659. args["bucketName"] = bucketName
  9660. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  9661. if err != nil {
  9662. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  9663. return
  9664. }
  9665. defer cleanupBucket(bucketName, c)
  9666. bufSize := dataFileMap["datatfile-33-kB"]
  9667. var reader = getDataReader("datafile-33-kB")
  9668. defer reader.Close()
  9669. objectName := fmt.Sprintf("test-file-%v", rand.Uint32())
  9670. args["objectName"] = objectName
  9671. ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
  9672. args["ctx"] = ctx
  9673. args["size"] = bufSize
  9674. defer cancel()
  9675. _, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  9676. if err != nil {
  9677. logError(testName, function, args, startTime, "", "PutObject with short timeout failed", err)
  9678. return
  9679. }
  9680. ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
  9681. args["ctx"] = ctx
  9682. defer cancel()
  9683. reader = getDataReader("datafile-33-kB")
  9684. defer reader.Close()
  9685. _, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  9686. if err != nil {
  9687. logError(testName, function, args, startTime, "", "PutObject with long timeout failed", err)
  9688. return
  9689. }
  9690. successLogger(testName, function, args, startTime).Info()
  9691. }
  9692. // Test get object with GetObject with custom context
  9693. func testGetObjectContextV2() {
  9694. // initialize logging params
  9695. startTime := time.Now()
  9696. testName := getFuncName()
  9697. function := "GetObject(ctx, bucketName, objectName)"
  9698. args := map[string]interface{}{
  9699. "ctx": "",
  9700. "bucketName": "",
  9701. "objectName": "",
  9702. }
  9703. // Seed random based on current time.
  9704. rand.Seed(time.Now().Unix())
  9705. // Instantiate new minio client object.
  9706. c, err := minio.New(os.Getenv(serverEndpoint),
  9707. &minio.Options{
  9708. Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  9709. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  9710. })
  9711. if err != nil {
  9712. logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
  9713. return
  9714. }
  9715. // Enable tracing, write to stderr.
  9716. // c.TraceOn(os.Stderr)
  9717. // Set user agent.
  9718. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  9719. // Generate a new random bucket name.
  9720. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  9721. args["bucketName"] = bucketName
  9722. // Make a new bucket.
  9723. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  9724. if err != nil {
  9725. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  9726. return
  9727. }
  9728. defer cleanupBucket(bucketName, c)
  9729. bufSize := dataFileMap["datafile-33-kB"]
  9730. var reader = getDataReader("datafile-33-kB")
  9731. defer reader.Close()
  9732. // Save the data
  9733. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  9734. args["objectName"] = objectName
  9735. _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  9736. if err != nil {
  9737. logError(testName, function, args, startTime, "", "PutObject call failed", err)
  9738. return
  9739. }
  9740. ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
  9741. args["ctx"] = ctx
  9742. cancel()
  9743. r, err := c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{})
  9744. if err != nil {
  9745. logError(testName, function, args, startTime, "", "GetObject failed unexpectedly", err)
  9746. return
  9747. }
  9748. if _, err = r.Stat(); err == nil {
  9749. logError(testName, function, args, startTime, "", "GetObject should fail on short timeout", err)
  9750. return
  9751. }
  9752. r.Close()
  9753. ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
  9754. defer cancel()
  9755. // Read the data back
  9756. r, err = c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{})
  9757. if err != nil {
  9758. logError(testName, function, args, startTime, "", "GetObject shouldn't fail on longer timeout", err)
  9759. return
  9760. }
  9761. st, err := r.Stat()
  9762. if err != nil {
  9763. logError(testName, function, args, startTime, "", "object Stat call failed", err)
  9764. return
  9765. }
  9766. if st.Size != int64(bufSize) {
  9767. logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(bufSize)+" got "+string(st.Size), err)
  9768. return
  9769. }
  9770. if err := r.Close(); err != nil {
  9771. logError(testName, function, args, startTime, "", " object Close() call failed", err)
  9772. return
  9773. }
  9774. successLogger(testName, function, args, startTime).Info()
  9775. }
  9776. // Test get object with FGetObject with custom context
  9777. func testFGetObjectContextV2() {
  9778. // initialize logging params
  9779. startTime := time.Now()
  9780. testName := getFuncName()
  9781. function := "FGetObject(ctx, bucketName, objectName,fileName)"
  9782. args := map[string]interface{}{
  9783. "ctx": "",
  9784. "bucketName": "",
  9785. "objectName": "",
  9786. "fileName": "",
  9787. }
  9788. // Seed random based on current time.
  9789. rand.Seed(time.Now().Unix())
  9790. // Instantiate new minio client object.
  9791. c, err := minio.New(os.Getenv(serverEndpoint),
  9792. &minio.Options{
  9793. Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  9794. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  9795. })
  9796. if err != nil {
  9797. logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
  9798. return
  9799. }
  9800. // Enable tracing, write to stderr.
  9801. // c.TraceOn(os.Stderr)
  9802. // Set user agent.
  9803. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  9804. // Generate a new random bucket name.
  9805. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  9806. args["bucketName"] = bucketName
  9807. // Make a new bucket.
  9808. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  9809. if err != nil {
  9810. logError(testName, function, args, startTime, "", "MakeBucket call failed", err)
  9811. return
  9812. }
  9813. defer cleanupBucket(bucketName, c)
  9814. bufSize := dataFileMap["datatfile-1-MB"]
  9815. var reader = getDataReader("datafile-1-MB")
  9816. defer reader.Close()
  9817. // Save the data
  9818. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  9819. args["objectName"] = objectName
  9820. _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
  9821. if err != nil {
  9822. logError(testName, function, args, startTime, "", "PutObject call failed", err)
  9823. return
  9824. }
  9825. ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
  9826. args["ctx"] = ctx
  9827. defer cancel()
  9828. fileName := "tempfile-context"
  9829. args["fileName"] = fileName
  9830. // Read the data back
  9831. err = c.FGetObject(ctx, bucketName, objectName, fileName+"-f", minio.GetObjectOptions{})
  9832. if err == nil {
  9833. logError(testName, function, args, startTime, "", "FGetObject should fail on short timeout", err)
  9834. return
  9835. }
  9836. ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
  9837. defer cancel()
  9838. // Read the data back
  9839. err = c.FGetObject(ctx, bucketName, objectName, fileName+"-fcontext", minio.GetObjectOptions{})
  9840. if err != nil {
  9841. logError(testName, function, args, startTime, "", "FGetObject call shouldn't fail on long timeout", err)
  9842. return
  9843. }
  9844. if err = os.Remove(fileName + "-fcontext"); err != nil {
  9845. logError(testName, function, args, startTime, "", "Remove file failed", err)
  9846. return
  9847. }
  9848. successLogger(testName, function, args, startTime).Info()
  9849. }
  9850. // Test list object v1 and V2
  9851. func testListObjects() {
  9852. // initialize logging params
  9853. startTime := time.Now()
  9854. testName := getFuncName()
  9855. function := "ListObjects(bucketName, objectPrefix, recursive, doneCh)"
  9856. args := map[string]interface{}{
  9857. "bucketName": "",
  9858. "objectPrefix": "",
  9859. "recursive": "true",
  9860. }
  9861. // Seed random based on current time.
  9862. rand.Seed(time.Now().Unix())
  9863. // Instantiate new minio client object.
  9864. c, err := minio.New(os.Getenv(serverEndpoint),
  9865. &minio.Options{
  9866. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  9867. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  9868. })
  9869. if err != nil {
  9870. logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err)
  9871. return
  9872. }
  9873. // Enable tracing, write to stderr.
  9874. // c.TraceOn(os.Stderr)
  9875. // Set user agent.
  9876. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  9877. // Generate a new random bucket name.
  9878. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  9879. args["bucketName"] = bucketName
  9880. // Make a new bucket.
  9881. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
  9882. if err != nil {
  9883. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  9884. return
  9885. }
  9886. defer cleanupBucket(bucketName, c)
  9887. testObjects := []struct {
  9888. name string
  9889. storageClass string
  9890. }{
  9891. // Special characters
  9892. {"foo bar", "STANDARD"},
  9893. {"foo-%", "STANDARD"},
  9894. {"random-object-1", "STANDARD"},
  9895. {"random-object-2", "REDUCED_REDUNDANCY"},
  9896. }
  9897. for i, object := range testObjects {
  9898. bufSize := dataFileMap["datafile-33-kB"]
  9899. var reader = getDataReader("datafile-33-kB")
  9900. defer reader.Close()
  9901. _, err = c.PutObject(context.Background(), bucketName, object.name, reader, int64(bufSize),
  9902. minio.PutObjectOptions{ContentType: "binary/octet-stream", StorageClass: object.storageClass})
  9903. if err != nil {
  9904. logError(testName, function, args, startTime, "", fmt.Sprintf("PutObject %d call failed", i+1), err)
  9905. return
  9906. }
  9907. }
  9908. testList := func(listFn func(context.Context, string, minio.ListObjectsOptions) <-chan minio.ObjectInfo, bucket string, opts minio.ListObjectsOptions) {
  9909. var objCursor int
  9910. // check for object name and storage-class from listing object result
  9911. for objInfo := range listFn(context.Background(), bucket, opts) {
  9912. if objInfo.Err != nil {
  9913. logError(testName, function, args, startTime, "", "ListObjects failed unexpectedly", err)
  9914. return
  9915. }
  9916. if objInfo.Key != testObjects[objCursor].name {
  9917. logError(testName, function, args, startTime, "", "ListObjects does not return expected object name", err)
  9918. return
  9919. }
  9920. if objInfo.StorageClass != testObjects[objCursor].storageClass {
  9921. // Ignored as Gateways (Azure/GCS etc) wont return storage class
  9922. ignoredLog(testName, function, args, startTime, "ListObjects doesn't return expected storage class").Info()
  9923. }
  9924. objCursor++
  9925. }
  9926. if objCursor != len(testObjects) {
  9927. logError(testName, function, args, startTime, "", "ListObjects returned unexpected number of items", errors.New(""))
  9928. return
  9929. }
  9930. }
  9931. testList(c.ListObjects, bucketName, minio.ListObjectsOptions{Recursive: true, UseV1: true})
  9932. testList(c.ListObjects, bucketName, minio.ListObjectsOptions{Recursive: true})
  9933. testList(c.ListObjects, bucketName, minio.ListObjectsOptions{Recursive: true, WithMetadata: true})
  9934. successLogger(testName, function, args, startTime).Info()
  9935. }
  9936. // Test deleting multiple objects with object retention set in Governance mode
  9937. func testRemoveObjects() {
  9938. // initialize logging params
  9939. startTime := time.Now()
  9940. testName := getFuncName()
  9941. function := "RemoveObjects(bucketName, objectsCh, opts)"
  9942. args := map[string]interface{}{
  9943. "bucketName": "",
  9944. "objectPrefix": "",
  9945. "recursive": "true",
  9946. }
  9947. // Seed random based on current time.
  9948. rand.Seed(time.Now().Unix())
  9949. // Instantiate new minio client object.
  9950. c, err := minio.New(os.Getenv(serverEndpoint),
  9951. &minio.Options{
  9952. Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
  9953. Secure: mustParseBool(os.Getenv(enableHTTPS)),
  9954. })
  9955. if err != nil {
  9956. logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err)
  9957. return
  9958. }
  9959. // Enable tracing, write to stderr.
  9960. // c.TraceOn(os.Stderr)
  9961. // Set user agent.
  9962. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
  9963. // Generate a new random bucket name.
  9964. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
  9965. args["bucketName"] = bucketName
  9966. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
  9967. args["objectName"] = objectName
  9968. // Make a new bucket.
  9969. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
  9970. if err != nil {
  9971. logError(testName, function, args, startTime, "", "MakeBucket failed", err)
  9972. return
  9973. }
  9974. bufSize := dataFileMap["datafile-129-MB"]
  9975. var reader = getDataReader("datafile-129-MB")
  9976. defer reader.Close()
  9977. _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{})
  9978. if err != nil {
  9979. logError(testName, function, args, startTime, "", "Error uploading object", err)
  9980. }
  9981. // Replace with smaller...
  9982. bufSize = dataFileMap["datafile-10-kB"]
  9983. reader = getDataReader("datafile-10-kB")
  9984. defer reader.Close()
  9985. _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{})
  9986. if err != nil {
  9987. logError(testName, function, args, startTime, "", "Error uploading object", err)
  9988. }
  9989. t := time.Date(2030, time.April, 25, 14, 0, 0, 0, time.UTC)
  9990. m := minio.RetentionMode(minio.Governance)
  9991. opts := minio.PutObjectRetentionOptions{
  9992. GovernanceBypass: false,
  9993. RetainUntilDate: &t,
  9994. Mode: &m,
  9995. }
  9996. err = c.PutObjectRetention(context.Background(), bucketName, objectName, opts)
  9997. if err != nil {
  9998. log.Fatalln(err)
  9999. }
  10000. objectsCh := make(chan minio.ObjectInfo)
  10001. // Send object names that are needed to be removed to objectsCh
  10002. go func() {
  10003. defer close(objectsCh)
  10004. // List all objects from a bucket-name with a matching prefix.
  10005. for object := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Recursive: true}) {
  10006. if object.Err != nil {
  10007. log.Fatalln(object.Err)
  10008. }
  10009. objectsCh <- object
  10010. }
  10011. }()
  10012. for rErr := range c.RemoveObjects(context.Background(), bucketName, objectsCh, minio.RemoveObjectsOptions{}) {
  10013. // Error is expected here because Retention is set on the object
  10014. // and RemoveObjects is called without Bypass Governance
  10015. if rErr.Err == nil {
  10016. logError(testName, function, args, startTime, "", "Expected error during deletion", nil)
  10017. return
  10018. }
  10019. }
  10020. objectsCh1 := make(chan minio.ObjectInfo)
  10021. // Send object names that are needed to be removed to objectsCh
  10022. go func() {
  10023. defer close(objectsCh1)
  10024. // List all objects from a bucket-name with a matching prefix.
  10025. for object := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Recursive: true}) {
  10026. if object.Err != nil {
  10027. log.Fatalln(object.Err)
  10028. }
  10029. objectsCh1 <- object
  10030. }
  10031. }()
  10032. opts1 := minio.RemoveObjectsOptions{
  10033. GovernanceBypass: true,
  10034. }
  10035. for rErr := range c.RemoveObjects(context.Background(), bucketName, objectsCh1, opts1) {
  10036. // Error is not expected here because Retention is set on the object
  10037. // and RemoveObjects is called with Bypass Governance
  10038. logError(testName, function, args, startTime, "", "Error detected during deletion", rErr.Err)
  10039. return
  10040. }
  10041. // Delete all objects and buckets
  10042. if err = cleanupVersionedBucket(bucketName, c); err != nil {
  10043. logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
  10044. return
  10045. }
  10046. successLogger(testName, function, args, startTime).Info()
  10047. }
  10048. // Convert string to bool and always return false if any error
  10049. func mustParseBool(str string) bool {
  10050. b, err := strconv.ParseBool(str)
  10051. if err != nil {
  10052. return false
  10053. }
  10054. return b
  10055. }
  10056. func main() {
  10057. // Output to stdout instead of the default stderr
  10058. log.SetOutput(os.Stdout)
  10059. // create custom formatter
  10060. mintFormatter := mintJSONFormatter{}
  10061. // set custom formatter
  10062. log.SetFormatter(&mintFormatter)
  10063. // log Info or above -- success cases are Info level, failures are Fatal level
  10064. log.SetLevel(log.InfoLevel)
  10065. tls := mustParseBool(os.Getenv(enableHTTPS))
  10066. kms := mustParseBool(os.Getenv(enableKMS))
  10067. if os.Getenv(enableKMS) == "" {
  10068. // Default to KMS tests.
  10069. kms = true
  10070. }
  10071. // execute tests
  10072. if isFullMode() {
  10073. testMakeBucketErrorV2()
  10074. testGetObjectClosedTwiceV2()
  10075. testFPutObjectV2()
  10076. testMakeBucketRegionsV2()
  10077. testGetObjectReadSeekFunctionalV2()
  10078. testGetObjectReadAtFunctionalV2()
  10079. testGetObjectRanges()
  10080. testCopyObjectV2()
  10081. testFunctionalV2()
  10082. testComposeObjectErrorCasesV2()
  10083. testCompose10KSourcesV2()
  10084. testUserMetadataCopyingV2()
  10085. testPutObject0ByteV2()
  10086. testPutObjectNoLengthV2()
  10087. testPutObjectsUnknownV2()
  10088. testGetObjectContextV2()
  10089. testFPutObjectContextV2()
  10090. testFGetObjectContextV2()
  10091. testPutObjectContextV2()
  10092. testPutObjectWithVersioning()
  10093. testMakeBucketError()
  10094. testMakeBucketRegions()
  10095. testPutObjectWithMetadata()
  10096. testPutObjectReadAt()
  10097. testPutObjectStreaming()
  10098. testGetObjectSeekEnd()
  10099. testGetObjectClosedTwice()
  10100. testRemoveMultipleObjects()
  10101. testFPutObjectMultipart()
  10102. testFPutObject()
  10103. testGetObjectReadSeekFunctional()
  10104. testGetObjectReadAtFunctional()
  10105. testGetObjectReadAtWhenEOFWasReached()
  10106. testPresignedPostPolicy()
  10107. testCopyObject()
  10108. testComposeObjectErrorCases()
  10109. testCompose10KSources()
  10110. testUserMetadataCopying()
  10111. testBucketNotification()
  10112. testFunctional()
  10113. testGetObjectModified()
  10114. testPutObjectUploadSeekedObject()
  10115. testGetObjectContext()
  10116. testFPutObjectContext()
  10117. testFGetObjectContext()
  10118. testGetObjectACLContext()
  10119. testPutObjectContext()
  10120. testStorageClassMetadataPutObject()
  10121. testStorageClassInvalidMetadataPutObject()
  10122. testStorageClassMetadataCopyObject()
  10123. testPutObjectWithContentLanguage()
  10124. testListObjects()
  10125. testRemoveObjects()
  10126. testListObjectVersions()
  10127. testStatObjectWithVersioning()
  10128. testGetObjectWithVersioning()
  10129. testCopyObjectWithVersioning()
  10130. testConcurrentCopyObjectWithVersioning()
  10131. testComposeObjectWithVersioning()
  10132. testRemoveObjectWithVersioning()
  10133. testRemoveObjectsWithVersioning()
  10134. testObjectTaggingWithVersioning()
  10135. // SSE-C tests will only work over TLS connection.
  10136. if tls {
  10137. testSSECEncryptionPutGet()
  10138. testSSECEncryptionFPut()
  10139. testSSECEncryptedGetObjectReadAtFunctional()
  10140. testSSECEncryptedGetObjectReadSeekFunctional()
  10141. testEncryptedCopyObjectV2()
  10142. testEncryptedSSECToSSECCopyObject()
  10143. testEncryptedSSECToUnencryptedCopyObject()
  10144. testUnencryptedToSSECCopyObject()
  10145. testUnencryptedToUnencryptedCopyObject()
  10146. testEncryptedEmptyObject()
  10147. testDecryptedCopyObject()
  10148. testSSECEncryptedToSSECCopyObjectPart()
  10149. testSSECMultipartEncryptedToSSECCopyObjectPart()
  10150. testSSECEncryptedToUnencryptedCopyPart()
  10151. testUnencryptedToSSECCopyObjectPart()
  10152. testUnencryptedToUnencryptedCopyPart()
  10153. testEncryptedSSECToSSES3CopyObject()
  10154. testEncryptedSSES3ToSSECCopyObject()
  10155. testSSECEncryptedToSSES3CopyObjectPart()
  10156. testSSES3EncryptedToSSECCopyObjectPart()
  10157. }
  10158. // KMS tests
  10159. if kms {
  10160. testSSES3EncryptionPutGet()
  10161. testSSES3EncryptionFPut()
  10162. testSSES3EncryptedGetObjectReadAtFunctional()
  10163. testSSES3EncryptedGetObjectReadSeekFunctional()
  10164. testEncryptedSSES3ToSSES3CopyObject()
  10165. testEncryptedSSES3ToUnencryptedCopyObject()
  10166. testUnencryptedToSSES3CopyObject()
  10167. testUnencryptedToSSES3CopyObjectPart()
  10168. testSSES3EncryptedToUnencryptedCopyPart()
  10169. testSSES3EncryptedToSSES3CopyObjectPart()
  10170. }
  10171. } else {
  10172. testFunctional()
  10173. testFunctionalV2()
  10174. }
  10175. }