1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906790779087909791079117912791379147915791679177918791979207921792279237924792579267927792879297930793179327933793479357936793779387939794079417942794379447945794679477948794979507951795279537954795579567957795879597960796179627963796479657966796779687969797079717972797379747975797679777978797979807981798279837984798579867987798879897990799179927993799479957996799779987999800080018002800380048005800680078008800980108011801280138014801580168017801880198020802180228023802480258026802780288029803080318032803380348035803680378038803980408041804280438044804580468047804880498050805180528053805480558056805780588059806080618062806380648065806680678068806980708071807280738074807580768077807880798080808180828083808480858086808780888089809080918092809380948095809680978098809981008101810281038104810581068107810881098110811181128113811481158116811781188119812081218122812381248125812681278128812981308131813281338134813581368137813881398140814181428143814481458146814781488149815081518152815381548155815681578158815981608161816281638164816581668167816881698170817181728173817481758176817781788179818081818182818381848185818681878188818981908191819281938194819581968197819881998200820182028203820482058206820782088209821082118212821382148215821682178218821982208221822282238224822582268227822882298230823182328233823482358236823782388239824082418242824382448245824682478248824982508251825282538254825582568257825882598260826182628263826482658266826782688269827082718272827382748275827682778278827982808281828282838284828582868287828882898290829182928293829482958296829782988299830083018302830383048305830683078308830983108311831283138314831583168317831883198320832183228323832483258326832783288329833083318332833383348335833683378338833983408341834283438344834583468347834883498350835183528353835483558356835783588359836083618362836383648365836683678368836983708371837283738374837583768377837883798380838183828383838483858386838783888389839083918392839383948395839683978398839984008401840284038404840584068407840884098410841184128413841484158416841784188419842084218422842384248425842684278428842984308431843284338434843584368437843884398440844184428443844484458446844784488449845084518452845384548455845684578458845984608461846284638464846584668467846884698470847184728473847484758476847784788479848084818482848384848485848684878488848984908491849284938494849584968497849884998500850185028503850485058506850785088509851085118512851385148515851685178518851985208521852285238524852585268527852885298530853185328533853485358536853785388539854085418542854385448545854685478548854985508551855285538554855585568557855885598560856185628563856485658566856785688569857085718572857385748575857685778578857985808581858285838584858585868587858885898590859185928593859485958596859785988599860086018602860386048605860686078608860986108611861286138614861586168617861886198620862186228623862486258626862786288629863086318632863386348635863686378638863986408641864286438644864586468647864886498650865186528653865486558656865786588659866086618662866386648665866686678668866986708671867286738674867586768677867886798680868186828683868486858686868786888689869086918692869386948695869686978698869987008701870287038704870587068707870887098710871187128713871487158716871787188719872087218722872387248725872687278728872987308731873287338734873587368737873887398740874187428743874487458746874787488749875087518752875387548755875687578758875987608761876287638764876587668767876887698770877187728773877487758776877787788779878087818782878387848785878687878788878987908791879287938794879587968797879887998800880188028803880488058806880788088809881088118812881388148815881688178818881988208821882288238824882588268827882888298830883188328833883488358836883788388839884088418842884388448845884688478848884988508851885288538854885588568857885888598860886188628863886488658866886788688869887088718872887388748875887688778878887988808881888288838884888588868887888888898890889188928893889488958896889788988899890089018902890389048905890689078908890989108911891289138914891589168917891889198920892189228923892489258926892789288929893089318932893389348935893689378938893989408941894289438944894589468947894889498950895189528953895489558956895789588959896089618962896389648965896689678968896989708971897289738974897589768977897889798980898189828983898489858986898789888989899089918992899389948995899689978998899990009001900290039004900590069007900890099010901190129013901490159016901790189019902090219022902390249025902690279028902990309031903290339034903590369037903890399040904190429043904490459046904790489049905090519052905390549055905690579058905990609061906290639064906590669067906890699070907190729073907490759076907790789079908090819082908390849085908690879088908990909091909290939094909590969097909890999100910191029103910491059106910791089109911091119112911391149115911691179118911991209121912291239124912591269127912891299130913191329133913491359136913791389139914091419142914391449145914691479148914991509151915291539154915591569157915891599160916191629163916491659166916791689169917091719172917391749175917691779178917991809181918291839184918591869187918891899190919191929193919491959196919791989199920092019202920392049205920692079208920992109211921292139214921592169217921892199220922192229223922492259226922792289229923092319232923392349235923692379238923992409241924292439244924592469247924892499250925192529253925492559256925792589259926092619262926392649265926692679268926992709271927292739274927592769277927892799280928192829283928492859286928792889289929092919292929392949295929692979298929993009301930293039304930593069307930893099310931193129313931493159316931793189319932093219322932393249325932693279328932993309331933293339334933593369337933893399340934193429343934493459346934793489349935093519352935393549355935693579358935993609361936293639364936593669367936893699370937193729373937493759376937793789379938093819382938393849385938693879388938993909391939293939394939593969397939893999400940194029403940494059406940794089409941094119412941394149415941694179418941994209421942294239424942594269427942894299430943194329433943494359436943794389439944094419442944394449445944694479448944994509451945294539454945594569457945894599460946194629463946494659466946794689469947094719472947394749475947694779478947994809481948294839484948594869487948894899490949194929493949494959496949794989499950095019502950395049505950695079508950995109511951295139514951595169517951895199520952195229523952495259526952795289529953095319532953395349535953695379538953995409541954295439544954595469547954895499550955195529553955495559556955795589559956095619562956395649565956695679568956995709571957295739574957595769577957895799580958195829583958495859586958795889589959095919592959395949595959695979598959996009601960296039604960596069607960896099610961196129613961496159616961796189619962096219622962396249625962696279628962996309631963296339634963596369637963896399640964196429643964496459646964796489649965096519652965396549655965696579658965996609661966296639664966596669667966896699670967196729673967496759676967796789679968096819682968396849685968696879688968996909691969296939694969596969697969896999700970197029703970497059706970797089709971097119712971397149715971697179718971997209721972297239724972597269727972897299730973197329733973497359736973797389739974097419742974397449745974697479748974997509751975297539754975597569757975897599760976197629763976497659766976797689769977097719772977397749775977697779778977997809781978297839784978597869787978897899790979197929793979497959796979797989799980098019802980398049805980698079808980998109811981298139814981598169817981898199820982198229823982498259826982798289829983098319832983398349835983698379838983998409841984298439844984598469847984898499850985198529853985498559856985798589859986098619862986398649865986698679868986998709871987298739874987598769877987898799880988198829883988498859886988798889889989098919892989398949895989698979898989999009901990299039904990599069907990899099910991199129913991499159916991799189919992099219922992399249925992699279928992999309931993299339934993599369937993899399940994199429943994499459946994799489949995099519952995399549955995699579958995999609961996299639964996599669967996899699970997199729973997499759976997799789979998099819982998399849985998699879988998999909991999299939994999599969997999899991000010001100021000310004100051000610007100081000910010100111001210013100141001510016100171001810019100201002110022100231002410025100261002710028100291003010031100321003310034100351003610037100381003910040100411004210043100441004510046100471004810049100501005110052100531005410055100561005710058100591006010061100621006310064100651006610067100681006910070100711007210073100741007510076100771007810079100801008110082100831008410085100861008710088100891009010091100921009310094100951009610097100981009910100101011010210103101041010510106101071010810109101101011110112101131011410115101161011710118101191012010121101221012310124101251012610127101281012910130101311013210133101341013510136101371013810139101401014110142101431014410145101461014710148101491015010151101521015310154101551015610157101581015910160101611016210163101641016510166101671016810169101701017110172101731017410175101761017710178101791018010181101821018310184101851018610187101881018910190101911019210193101941019510196101971019810199102001020110202102031020410205102061020710208102091021010211102121021310214102151021610217102181021910220102211022210223102241022510226102271022810229102301023110232102331023410235102361023710238102391024010241102421024310244102451024610247102481024910250102511025210253102541025510256102571025810259102601026110262102631026410265102661026710268102691027010271102721027310274102751027610277102781027910280102811028210283102841028510286102871028810289102901029110292102931029410295102961029710298102991030010301103021030310304103051030610307103081030910310103111031210313103141031510316103171031810319103201032110322103231032410325103261032710328103291033010331103321033310334103351033610337103381033910340103411034210343103441034510346103471034810349103501035110352103531035410355103561035710358103591036010361103621036310364103651036610367103681036910370103711037210373103741037510376103771037810379103801038110382103831038410385103861038710388103891039010391103921039310394103951039610397103981039910400104011040210403104041040510406104071040810409104101041110412104131041410415104161041710418104191042010421104221042310424104251042610427104281042910430104311043210433104341043510436104371043810439104401044110442104431044410445104461044710448104491045010451104521045310454104551045610457104581045910460104611046210463104641046510466104671046810469104701047110472104731047410475104761047710478104791048010481104821048310484104851048610487104881048910490104911049210493104941049510496104971049810499105001050110502105031050410505105061050710508105091051010511105121051310514105151051610517105181051910520105211052210523105241052510526105271052810529105301053110532105331053410535105361053710538105391054010541105421054310544105451054610547105481054910550105511055210553105541055510556105571055810559105601056110562105631056410565105661056710568105691057010571105721057310574105751057610577105781057910580105811058210583105841058510586105871058810589105901059110592105931059410595105961059710598105991060010601106021060310604106051060610607106081060910610106111061210613106141061510616106171061810619106201062110622106231062410625106261062710628106291063010631106321063310634106351063610637106381063910640106411064210643106441064510646106471064810649106501065110652106531065410655106561065710658106591066010661106621066310664106651066610667106681066910670106711067210673106741067510676106771067810679106801068110682106831068410685106861068710688106891069010691106921069310694106951069610697106981069910700107011070210703107041070510706107071070810709107101071110712107131071410715107161071710718107191072010721107221072310724107251072610727107281072910730107311073210733107341073510736107371073810739107401074110742107431074410745107461074710748107491075010751107521075310754107551075610757107581075910760107611076210763107641076510766107671076810769107701077110772107731077410775107761077710778107791078010781107821078310784107851078610787107881078910790107911079210793107941079510796107971079810799108001080110802108031080410805108061080710808108091081010811108121081310814108151081610817108181081910820108211082210823108241082510826108271082810829108301083110832108331083410835108361083710838108391084010841108421084310844108451084610847108481084910850108511085210853108541085510856108571085810859108601086110862108631086410865108661086710868108691087010871108721087310874108751087610877108781087910880108811088210883108841088510886108871088810889108901089110892108931089410895108961089710898108991090010901109021090310904109051090610907109081090910910109111091210913109141091510916109171091810919109201092110922109231092410925109261092710928109291093010931109321093310934109351093610937109381093910940109411094210943109441094510946109471094810949109501095110952109531095410955109561095710958109591096010961109621096310964109651096610967109681096910970109711097210973109741097510976109771097810979109801098110982109831098410985109861098710988109891099010991109921099310994109951099610997109981099911000110011100211003110041100511006110071100811009110101101111012110131101411015110161101711018110191102011021110221102311024110251102611027110281102911030110311103211033110341103511036110371103811039110401104111042110431104411045110461104711048110491105011051110521105311054110551105611057110581105911060110611106211063110641106511066110671106811069110701107111072110731107411075110761107711078110791108011081110821108311084110851108611087110881108911090110911109211093110941109511096110971109811099111001110111102111031110411105111061110711108111091111011111111121111311114111151111611117111181111911120111211112211123111241112511126111271112811129111301113111132111331113411135111361113711138111391114011141111421114311144111451114611147111481114911150111511115211153111541115511156111571115811159111601116111162111631116411165111661116711168111691117011171111721117311174111751117611177111781117911180111811118211183111841118511186111871118811189111901119111192111931119411195111961119711198111991120011201112021120311204112051120611207112081120911210112111121211213112141121511216112171121811219112201122111222112231122411225112261122711228112291123011231112321123311234112351123611237112381123911240112411124211243112441124511246112471124811249112501125111252112531125411255112561125711258112591126011261112621126311264112651126611267112681126911270112711127211273112741127511276112771127811279112801128111282112831128411285112861128711288112891129011291112921129311294112951129611297112981129911300113011130211303113041130511306113071130811309113101131111312113131131411315113161131711318113191132011321113221132311324113251132611327113281132911330113311133211333113341133511336113371133811339113401134111342113431134411345113461134711348113491135011351113521135311354113551135611357113581135911360113611136211363113641136511366113671136811369113701137111372113731137411375113761137711378113791138011381113821138311384113851138611387113881138911390113911139211393113941139511396113971139811399114001140111402114031140411405114061140711408114091141011411114121141311414114151141611417114181141911420114211142211423114241142511426114271142811429114301143111432114331143411435114361143711438114391144011441114421144311444114451144611447114481144911450114511145211453114541145511456114571145811459114601146111462114631146411465114661146711468114691147011471114721147311474114751147611477114781147911480114811148211483114841148511486114871148811489114901149111492114931149411495114961149711498114991150011501115021150311504115051150611507115081150911510115111151211513115141151511516115171151811519115201152111522115231152411525115261152711528115291153011531115321153311534115351153611537115381153911540115411154211543115441154511546115471154811549115501155111552115531155411555115561155711558115591156011561115621156311564115651156611567115681156911570115711157211573115741157511576115771157811579115801158111582115831158411585115861158711588115891159011591115921159311594115951159611597115981159911600116011160211603116041160511606116071160811609116101161111612116131161411615116161161711618116191162011621116221162311624116251162611627116281162911630116311163211633116341163511636116371163811639116401164111642116431164411645116461164711648116491165011651116521165311654116551165611657116581165911660116611166211663116641166511666116671166811669116701167111672116731167411675116761167711678116791168011681116821168311684116851168611687116881168911690116911169211693116941169511696116971169811699117001170111702117031170411705117061170711708117091171011711117121171311714117151171611717117181171911720117211172211723117241172511726117271172811729117301173111732117331173411735117361173711738117391174011741117421174311744117451174611747117481174911750117511175211753117541175511756117571175811759117601176111762117631176411765117661176711768117691177011771117721177311774117751177611777117781177911780117811178211783117841178511786117871178811789117901179111792117931179411795117961179711798117991180011801118021180311804118051180611807118081180911810 |
- // +build mint
- /*
- * MinIO Go Library for Amazon S3 Compatible Cloud Storage
- * Copyright 2015-2020 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
- package main
- import (
- "bytes"
- "context"
- "errors"
- "fmt"
- "hash/crc32"
- "io"
- "io/ioutil"
- "math/rand"
- "mime/multipart"
- "net/http"
- "net/url"
- "os"
- "path/filepath"
- "reflect"
- "runtime"
- "sort"
- "strconv"
- "strings"
- "sync"
- "time"
- "github.com/dustin/go-humanize"
- jsoniter "github.com/json-iterator/go"
- log "github.com/sirupsen/logrus"
- "github.com/minio/minio-go/v7"
- "github.com/minio/minio-go/v7/pkg/credentials"
- "github.com/minio/minio-go/v7/pkg/encrypt"
- "github.com/minio/minio-go/v7/pkg/notification"
- "github.com/minio/minio-go/v7/pkg/tags"
- )
- const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569"
- const (
- letterIdxBits = 6 // 6 bits to represent a letter index
- letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
- letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits
- )
- const (
- serverEndpoint = "SERVER_ENDPOINT"
- accessKey = "ACCESS_KEY"
- secretKey = "SECRET_KEY"
- enableHTTPS = "ENABLE_HTTPS"
- enableKMS = "ENABLE_KMS"
- )
- type mintJSONFormatter struct {
- }
- func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) {
- data := make(log.Fields, len(entry.Data))
- for k, v := range entry.Data {
- switch v := v.(type) {
- case error:
- // Otherwise errors are ignored by `encoding/json`
- // https://github.com/sirupsen/logrus/issues/137
- data[k] = v.Error()
- default:
- data[k] = v
- }
- }
- var json = jsoniter.ConfigCompatibleWithStandardLibrary
- serialized, err := json.Marshal(data)
- if err != nil {
- return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err)
- }
- return append(serialized, '\n'), nil
- }
- var readFull = func(r io.Reader, buf []byte) (n int, err error) {
- // ReadFull reads exactly len(buf) bytes from r into buf.
- // It returns the number of bytes copied and an error if
- // fewer bytes were read. The error is EOF only if no bytes
- // were read. If an EOF happens after reading some but not
- // all the bytes, ReadFull returns ErrUnexpectedEOF.
- // On return, n == len(buf) if and only if err == nil.
- // If r returns an error having read at least len(buf) bytes,
- // the error is dropped.
- for n < len(buf) && err == nil {
- var nn int
- nn, err = r.Read(buf[n:])
- // Some spurious io.Reader's return
- // io.ErrUnexpectedEOF when nn == 0
- // this behavior is undocumented
- // so we are on purpose not using io.ReadFull
- // implementation because this can lead
- // to custom handling, to avoid that
- // we simply modify the original io.ReadFull
- // implementation to avoid this issue.
- // io.ErrUnexpectedEOF with nn == 0 really
- // means that io.EOF
- if err == io.ErrUnexpectedEOF && nn == 0 {
- err = io.EOF
- }
- n += nn
- }
- if n >= len(buf) {
- err = nil
- } else if n > 0 && err == io.EOF {
- err = io.ErrUnexpectedEOF
- }
- return
- }
- func cleanEmptyEntries(fields log.Fields) log.Fields {
- cleanFields := log.Fields{}
- for k, v := range fields {
- if v != "" {
- cleanFields[k] = v
- }
- }
- return cleanFields
- }
- // log successful test runs
- func successLogger(testName string, function string, args map[string]interface{}, startTime time.Time) *log.Entry {
- // calculate the test case duration
- duration := time.Since(startTime)
- // log with the fields as per mint
- fields := log.Fields{"name": "minio-go: " + testName, "function": function, "args": args, "duration": duration.Nanoseconds() / 1000000, "status": "PASS"}
- return log.WithFields(cleanEmptyEntries(fields))
- }
- // As few of the features are not available in Gateway(s) currently, Check if err value is NotImplemented,
- // and log as NA in that case and continue execution. Otherwise log as failure and return
- func logError(testName string, function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) {
- // If server returns NotImplemented we assume it is gateway mode and hence log it as info and move on to next tests
- // Special case for ComposeObject API as it is implemented on client side and adds specific error details like `Error in upload-part-copy` in
- // addition to NotImplemented error returned from server
- if isErrNotImplemented(err) {
- ignoredLog(testName, function, args, startTime, message).Info()
- } else {
- failureLog(testName, function, args, startTime, alert, message, err).Fatal()
- }
- }
- // log failed test runs
- func failureLog(testName string, function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry {
- // calculate the test case duration
- duration := time.Since(startTime)
- var fields log.Fields
- // log with the fields as per mint
- if err != nil {
- fields = log.Fields{"name": "minio-go: " + testName, "function": function, "args": args,
- "duration": duration.Nanoseconds() / 1000000, "status": "FAIL", "alert": alert, "message": message, "error": err}
- } else {
- fields = log.Fields{"name": "minio-go: " + testName, "function": function, "args": args,
- "duration": duration.Nanoseconds() / 1000000, "status": "FAIL", "alert": alert, "message": message}
- }
- return log.WithFields(cleanEmptyEntries(fields))
- }
- // log not applicable test runs
- func ignoredLog(testName string, function string, args map[string]interface{}, startTime time.Time, alert string) *log.Entry {
- // calculate the test case duration
- duration := time.Since(startTime)
- // log with the fields as per mint
- fields := log.Fields{"name": "minio-go: " + testName, "function": function, "args": args,
- "duration": duration.Nanoseconds() / 1000000, "status": "NA", "alert": strings.Split(alert, " ")[0] + " is NotImplemented"}
- return log.WithFields(cleanEmptyEntries(fields))
- }
- // Delete objects in given bucket, recursively
- func cleanupBucket(bucketName string, c *minio.Client) error {
- // Create a done channel to control 'ListObjectsV2' go routine.
- doneCh := make(chan struct{})
- // Exit cleanly upon return.
- defer close(doneCh)
- // Iterate over all objects in the bucket via listObjectsV2 and delete
- for objCh := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{Recursive: true}) {
- if objCh.Err != nil {
- return objCh.Err
- }
- if objCh.Key != "" {
- err := c.RemoveObject(context.Background(), bucketName, objCh.Key, minio.RemoveObjectOptions{})
- if err != nil {
- return err
- }
- }
- }
- for objPartInfo := range c.ListIncompleteUploads(context.Background(), bucketName, "", true) {
- if objPartInfo.Err != nil {
- return objPartInfo.Err
- }
- if objPartInfo.Key != "" {
- err := c.RemoveIncompleteUpload(context.Background(), bucketName, objPartInfo.Key)
- if err != nil {
- return err
- }
- }
- }
- // objects are already deleted, clear the buckets now
- err := c.RemoveBucket(context.Background(), bucketName)
- if err != nil {
- return err
- }
- return err
- }
- func cleanupVersionedBucket(bucketName string, c *minio.Client) error {
- doneCh := make(chan struct{})
- defer close(doneCh)
- for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) {
- if obj.Err != nil {
- return obj.Err
- }
- if obj.Key != "" {
- err := c.RemoveObject(context.Background(), bucketName, obj.Key,
- minio.RemoveObjectOptions{VersionID: obj.VersionID, GovernanceBypass: true})
- if err != nil {
- return err
- }
- }
- }
- for objPartInfo := range c.ListIncompleteUploads(context.Background(), bucketName, "", true) {
- if objPartInfo.Err != nil {
- return objPartInfo.Err
- }
- if objPartInfo.Key != "" {
- err := c.RemoveIncompleteUpload(context.Background(), bucketName, objPartInfo.Key)
- if err != nil {
- return err
- }
- }
- }
- // objects are already deleted, clear the buckets now
- err := c.RemoveBucket(context.Background(), bucketName)
- if err != nil {
- for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) {
- log.Println("found", obj.Key, obj.VersionID)
- }
- return err
- }
- return err
- }
- func isErrNotImplemented(err error) bool {
- return minio.ToErrorResponse(err).Code == "NotImplemented"
- }
- func init() {
- // If server endpoint is not set, all tests default to
- // using https://play.min.io
- if os.Getenv(serverEndpoint) == "" {
- os.Setenv(serverEndpoint, "play.min.io")
- os.Setenv(accessKey, "Q3AM3UQ867SPQQA43P2F")
- os.Setenv(secretKey, "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG")
- os.Setenv(enableHTTPS, "1")
- }
- }
- var mintDataDir = os.Getenv("MINT_DATA_DIR")
- func getMintDataDirFilePath(filename string) (fp string) {
- if mintDataDir == "" {
- return
- }
- return filepath.Join(mintDataDir, filename)
- }
- func newRandomReader(seed, size int64) io.Reader {
- return io.LimitReader(rand.New(rand.NewSource(seed)), size)
- }
- func mustCrcReader(r io.Reader) uint32 {
- crc := crc32.NewIEEE()
- _, err := io.Copy(crc, r)
- if err != nil {
- panic(err)
- }
- return crc.Sum32()
- }
- func crcMatches(r io.Reader, want uint32) error {
- crc := crc32.NewIEEE()
- _, err := io.Copy(crc, r)
- if err != nil {
- panic(err)
- }
- got := crc.Sum32()
- if got != want {
- return fmt.Errorf("crc mismatch, want %x, got %x", want, got)
- }
- return nil
- }
- func crcMatchesName(r io.Reader, name string) error {
- want := dataFileCRC32[name]
- crc := crc32.NewIEEE()
- _, err := io.Copy(crc, r)
- if err != nil {
- panic(err)
- }
- got := crc.Sum32()
- if got != want {
- return fmt.Errorf("crc mismatch, want %x, got %x", want, got)
- }
- return nil
- }
- // read data from file if it exists or optionally create a buffer of particular size
- func getDataReader(fileName string) io.ReadCloser {
- if mintDataDir == "" {
- size := int64(dataFileMap[fileName])
- if _, ok := dataFileCRC32[fileName]; !ok {
- dataFileCRC32[fileName] = mustCrcReader(newRandomReader(size, size))
- }
- return ioutil.NopCloser(newRandomReader(size, size))
- }
- reader, _ := os.Open(getMintDataDirFilePath(fileName))
- if _, ok := dataFileCRC32[fileName]; !ok {
- dataFileCRC32[fileName] = mustCrcReader(reader)
- reader.Close()
- reader, _ = os.Open(getMintDataDirFilePath(fileName))
- }
- return reader
- }
- // randString generates random names and prepends them with a known prefix.
- func randString(n int, src rand.Source, prefix string) string {
- b := make([]byte, n)
- // A rand.Int63() generates 63 random bits, enough for letterIdxMax letters!
- for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
- if remain == 0 {
- cache, remain = src.Int63(), letterIdxMax
- }
- if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
- b[i] = letterBytes[idx]
- i--
- }
- cache >>= letterIdxBits
- remain--
- }
- return prefix + string(b[0:30-len(prefix)])
- }
- var dataFileMap = map[string]int{
- "datafile-0-b": 0,
- "datafile-1-b": 1,
- "datafile-1-kB": 1 * humanize.KiByte,
- "datafile-10-kB": 10 * humanize.KiByte,
- "datafile-33-kB": 33 * humanize.KiByte,
- "datafile-100-kB": 100 * humanize.KiByte,
- "datafile-1.03-MB": 1056 * humanize.KiByte,
- "datafile-1-MB": 1 * humanize.MiByte,
- "datafile-5-MB": 5 * humanize.MiByte,
- "datafile-6-MB": 6 * humanize.MiByte,
- "datafile-11-MB": 11 * humanize.MiByte,
- "datafile-65-MB": 65 * humanize.MiByte,
- "datafile-129-MB": 129 * humanize.MiByte,
- }
- var dataFileCRC32 = map[string]uint32{}
- func isFullMode() bool {
- return os.Getenv("MINT_MODE") == "full"
- }
- func getFuncName() string {
- return getFuncNameLoc(2)
- }
- func getFuncNameLoc(caller int) string {
- pc, _, _, _ := runtime.Caller(caller)
- return strings.TrimPrefix(runtime.FuncForPC(pc).Name(), "main.")
- }
- // Tests bucket re-create errors.
- func testMakeBucketError() {
- region := "eu-central-1"
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "MakeBucket(bucketName, region)"
- // initialize logging params
- args := map[string]interface{}{
- "bucketName": "",
- "region": region,
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket in 'eu-central-1'.
- if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket Failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err == nil {
- logError(testName, function, args, startTime, "", "Bucket already exists", err)
- return
- }
- // Verify valid error response from server.
- if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" &&
- minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" {
- logError(testName, function, args, startTime, "", "Invalid error returned by server", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- func testMetadataSizeLimit() {
- startTime := time.Now()
- testName := getFuncName()
- function := "PutObject(bucketName, objectName, reader, objectSize, opts)"
- args := map[string]interface{}{
- "bucketName": "",
- "objectName": "",
- "opts.UserMetadata": "",
- }
- rand.Seed(startTime.Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client creation failed", err)
- return
- }
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "Make bucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- const HeaderSizeLimit = 8 * 1024
- const UserMetadataLimit = 2 * 1024
- // Meta-data greater than the 2 KB limit of AWS - PUT calls with this meta-data should fail
- metadata := make(map[string]string)
- metadata["X-Amz-Meta-Mint-Test"] = string(bytes.Repeat([]byte("m"), 1+UserMetadataLimit-len("X-Amz-Meta-Mint-Test")))
- args["metadata"] = fmt.Sprint(metadata)
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata})
- if err == nil {
- logError(testName, function, args, startTime, "", "Created object with user-defined metadata exceeding metadata size limits", nil)
- return
- }
- // Meta-data (headers) greater than the 8 KB limit of AWS - PUT calls with this meta-data should fail
- metadata = make(map[string]string)
- metadata["X-Amz-Mint-Test"] = string(bytes.Repeat([]byte("m"), 1+HeaderSizeLimit-len("X-Amz-Mint-Test")))
- args["metadata"] = fmt.Sprint(metadata)
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata})
- if err == nil {
- logError(testName, function, args, startTime, "", "Created object with headers exceeding header size limits", nil)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Tests various bucket supported formats.
- func testMakeBucketRegions() {
- region := "eu-central-1"
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "MakeBucket(bucketName, region)"
- // initialize logging params
- args := map[string]interface{}{
- "bucketName": "",
- "region": region,
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket in 'eu-central-1'.
- if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- // Delete all objects and buckets
- if err = cleanupBucket(bucketName, c); err != nil {
- logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
- return
- }
- // Make a new bucket with '.' in its name, in 'us-west-2'. This
- // request is internally staged into a path style instead of
- // virtual host style.
- region = "us-west-2"
- args["region"] = region
- if err = c.MakeBucket(context.Background(), bucketName+".withperiod", minio.MakeBucketOptions{Region: region}); err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- // Delete all objects and buckets
- if err = cleanupBucket(bucketName+".withperiod", c); err != nil {
- logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test PutObject using a large data to trigger multipart readat
- func testPutObjectReadAt() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "PutObject(bucketName, objectName, reader, opts)"
- args := map[string]interface{}{
- "bucketName": "",
- "objectName": "",
- "opts": "objectContentType",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "Make bucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- bufSize := dataFileMap["datafile-129-MB"]
- var reader = getDataReader("datafile-129-MB")
- defer reader.Close()
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- // Object content type
- objectContentType := "binary/octet-stream"
- args["objectContentType"] = objectContentType
- _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: objectContentType})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- // Read the data back
- r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "Get Object failed", err)
- return
- }
- st, err := r.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat Object failed", err)
- return
- }
- if st.Size != int64(bufSize) {
- logError(testName, function, args, startTime, "", fmt.Sprintf("Number of bytes in stat does not match, expected %d got %d", bufSize, st.Size), err)
- return
- }
- if st.ContentType != objectContentType && st.ContentType != "application/octet-stream" {
- logError(testName, function, args, startTime, "", "Content types don't match", err)
- return
- }
- if err := crcMatchesName(r, "datafile-129-MB"); err != nil {
- logError(testName, function, args, startTime, "", "data CRC check failed", err)
- return
- }
- if err := r.Close(); err != nil {
- logError(testName, function, args, startTime, "", "Object Close failed", err)
- return
- }
- if err := r.Close(); err == nil {
- logError(testName, function, args, startTime, "", "Object is already closed, didn't return error on Close", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- func testListObjectVersions() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "ListObjectVersions(bucketName, prefix, recursive)"
- args := map[string]interface{}{
- "bucketName": "",
- "prefix": "",
- "recursive": "",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
- if err != nil {
- logError(testName, function, args, startTime, "", "Make bucket failed", err)
- return
- }
- err = c.EnableVersioning(context.Background(), bucketName)
- if err != nil {
- logError(testName, function, args, startTime, "", "Enable versioning failed", err)
- return
- }
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- bufSize := dataFileMap["datafile-10-kB"]
- var reader = getDataReader("datafile-10-kB")
- _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- reader.Close()
- bufSize = dataFileMap["datafile-1-b"]
- reader = getDataReader("datafile-1-b")
- _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- reader.Close()
- err = c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "Unexpected object deletion", err)
- return
- }
- var deleteMarkers, versions int
- objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
- for info := range objectsInfo {
- if info.Err != nil {
- logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
- return
- }
- if info.Key != objectName {
- logError(testName, function, args, startTime, "", "Unexpected object name in listing objects", nil)
- return
- }
- if info.VersionID == "" {
- logError(testName, function, args, startTime, "", "Unexpected version id in listing objects", nil)
- return
- }
- if info.IsDeleteMarker {
- deleteMarkers++
- if !info.IsLatest {
- logError(testName, function, args, startTime, "", "Unexpected IsLatest field in listing objects", nil)
- return
- }
- } else {
- versions++
- }
- }
- if deleteMarkers != 1 {
- logError(testName, function, args, startTime, "", "Unexpected number of DeleteMarker elements in listing objects", nil)
- return
- }
- if versions != 2 {
- logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil)
- return
- }
- // Delete all objects and their versions as long as the bucket itself
- if err = cleanupVersionedBucket(bucketName, c); err != nil {
- logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- func testStatObjectWithVersioning() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "StatObject"
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
- if err != nil {
- logError(testName, function, args, startTime, "", "Make bucket failed", err)
- return
- }
- err = c.EnableVersioning(context.Background(), bucketName)
- if err != nil {
- logError(testName, function, args, startTime, "", "Enable versioning failed", err)
- return
- }
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- bufSize := dataFileMap["datafile-10-kB"]
- var reader = getDataReader("datafile-10-kB")
- _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- reader.Close()
- bufSize = dataFileMap["datafile-1-b"]
- reader = getDataReader("datafile-1-b")
- _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- reader.Close()
- objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
- var results []minio.ObjectInfo
- for info := range objectsInfo {
- if info.Err != nil {
- logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
- return
- }
- results = append(results, info)
- }
- if len(results) != 2 {
- logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil)
- return
- }
- for i := 0; i < len(results); i++ {
- opts := minio.StatObjectOptions{VersionID: results[i].VersionID}
- statInfo, err := c.StatObject(context.Background(), bucketName, objectName, opts)
- if err != nil {
- logError(testName, function, args, startTime, "", "error during HEAD object", err)
- return
- }
- if statInfo.VersionID == "" || statInfo.VersionID != results[i].VersionID {
- logError(testName, function, args, startTime, "", "error during HEAD object, unexpected version id", err)
- return
- }
- if statInfo.ETag != results[i].ETag {
- logError(testName, function, args, startTime, "", "error during HEAD object, unexpected ETag", err)
- return
- }
- if statInfo.LastModified.Unix() != results[i].LastModified.Unix() {
- logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Last-Modified", err)
- return
- }
- if statInfo.Size != results[i].Size {
- logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Content-Length", err)
- return
- }
- }
- // Delete all objects and their versions as long as the bucket itself
- if err = cleanupVersionedBucket(bucketName, c); err != nil {
- logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- func testGetObjectWithVersioning() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "GetObject()"
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
- if err != nil {
- logError(testName, function, args, startTime, "", "Make bucket failed", err)
- return
- }
- err = c.EnableVersioning(context.Background(), bucketName)
- if err != nil {
- logError(testName, function, args, startTime, "", "Enable versioning failed", err)
- return
- }
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- // Save the contents of datafiles to check with GetObject() reader output later
- var buffers [][]byte
- var testFiles = []string{"datafile-1-b", "datafile-10-kB"}
- for _, testFile := range testFiles {
- r := getDataReader(testFile)
- buf, err := ioutil.ReadAll(r)
- if err != nil {
- logError(testName, function, args, startTime, "", "unexpected failure", err)
- return
- }
- r.Close()
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- buffers = append(buffers, buf)
- }
- objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
- var results []minio.ObjectInfo
- for info := range objectsInfo {
- if info.Err != nil {
- logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
- return
- }
- results = append(results, info)
- }
- if len(results) != 2 {
- logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil)
- return
- }
- sort.SliceStable(results, func(i, j int) bool {
- return results[i].Size < results[j].Size
- })
- sort.SliceStable(buffers, func(i, j int) bool {
- return len(buffers[i]) < len(buffers[j])
- })
- for i := 0; i < len(results); i++ {
- opts := minio.GetObjectOptions{VersionID: results[i].VersionID}
- reader, err := c.GetObject(context.Background(), bucketName, objectName, opts)
- if err != nil {
- logError(testName, function, args, startTime, "", "error during GET object", err)
- return
- }
- statInfo, err := reader.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "error during calling reader.Stat()", err)
- return
- }
- if statInfo.ETag != results[i].ETag {
- logError(testName, function, args, startTime, "", "error during HEAD object, unexpected ETag", err)
- return
- }
- if statInfo.LastModified.Unix() != results[i].LastModified.Unix() {
- logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Last-Modified", err)
- return
- }
- if statInfo.Size != results[i].Size {
- logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Content-Length", err)
- return
- }
- tmpBuffer := bytes.NewBuffer([]byte{})
- _, err = io.Copy(tmpBuffer, reader)
- if err != nil {
- logError(testName, function, args, startTime, "", "unexpected io.Copy()", err)
- return
- }
- if !bytes.Equal(tmpBuffer.Bytes(), buffers[i]) {
- logError(testName, function, args, startTime, "", "unexpected content of GetObject()", err)
- return
- }
- }
- // Delete all objects and their versions as long as the bucket itself
- if err = cleanupVersionedBucket(bucketName, c); err != nil {
- logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- func testPutObjectWithVersioning() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "GetObject()"
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
- if err != nil {
- logError(testName, function, args, startTime, "", "Make bucket failed", err)
- return
- }
- err = c.EnableVersioning(context.Background(), bucketName)
- if err != nil {
- logError(testName, function, args, startTime, "", "Enable versioning failed", err)
- return
- }
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- const n = 10
- // Read input...
- // Save the data concurrently.
- var wg sync.WaitGroup
- wg.Add(n)
- var buffers = make([][]byte, n)
- var errs [n]error
- for i := 0; i < n; i++ {
- r := newRandomReader(int64((1<<20)*i+i), int64(i))
- buf, err := ioutil.ReadAll(r)
- if err != nil {
- logError(testName, function, args, startTime, "", "unexpected failure", err)
- return
- }
- buffers[i] = buf
- go func(i int) {
- defer wg.Done()
- _, errs[i] = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{PartSize: 5 << 20})
- }(i)
- }
- wg.Wait()
- for _, err := range errs {
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- }
- objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
- var results []minio.ObjectInfo
- for info := range objectsInfo {
- if info.Err != nil {
- logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
- return
- }
- results = append(results, info)
- }
- if len(results) != n {
- logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil)
- return
- }
- sort.Slice(results, func(i, j int) bool {
- return results[i].Size < results[j].Size
- })
- sort.Slice(buffers, func(i, j int) bool {
- return len(buffers[i]) < len(buffers[j])
- })
- for i := 0; i < len(results); i++ {
- opts := minio.GetObjectOptions{VersionID: results[i].VersionID}
- reader, err := c.GetObject(context.Background(), bucketName, objectName, opts)
- if err != nil {
- logError(testName, function, args, startTime, "", "error during GET object", err)
- return
- }
- statInfo, err := reader.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "error during calling reader.Stat()", err)
- return
- }
- if statInfo.ETag != results[i].ETag {
- logError(testName, function, args, startTime, "", "error during HEAD object, unexpected ETag", err)
- return
- }
- if statInfo.LastModified.Unix() != results[i].LastModified.Unix() {
- logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Last-Modified", err)
- return
- }
- if statInfo.Size != results[i].Size {
- logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Content-Length", err)
- return
- }
- tmpBuffer := bytes.NewBuffer([]byte{})
- _, err = io.Copy(tmpBuffer, reader)
- if err != nil {
- logError(testName, function, args, startTime, "", "unexpected io.Copy()", err)
- return
- }
- if !bytes.Equal(tmpBuffer.Bytes(), buffers[i]) {
- logError(testName, function, args, startTime, "", "unexpected content of GetObject()", err)
- return
- }
- }
- // Delete all objects and their versions as long as the bucket itself
- if err = cleanupVersionedBucket(bucketName, c); err != nil {
- logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- func testCopyObjectWithVersioning() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObject()"
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
- if err != nil {
- logError(testName, function, args, startTime, "", "Make bucket failed", err)
- return
- }
- err = c.EnableVersioning(context.Background(), bucketName)
- if err != nil {
- logError(testName, function, args, startTime, "", "Enable versioning failed", err)
- return
- }
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- var testFiles = []string{"datafile-1-b", "datafile-10-kB"}
- for _, testFile := range testFiles {
- r := getDataReader(testFile)
- buf, err := ioutil.ReadAll(r)
- if err != nil {
- logError(testName, function, args, startTime, "", "unexpected failure", err)
- return
- }
- r.Close()
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- }
- objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
- var infos []minio.ObjectInfo
- for info := range objectsInfo {
- if info.Err != nil {
- logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
- return
- }
- infos = append(infos, info)
- }
- sort.Slice(infos, func(i, j int) bool {
- return infos[i].Size < infos[j].Size
- })
- reader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{VersionID: infos[0].VersionID})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject of the oldest version content failed", err)
- return
- }
- oldestContent, err := ioutil.ReadAll(reader)
- if err != nil {
- logError(testName, function, args, startTime, "", "Reading the oldest object version failed", err)
- return
- }
- // Copy Source
- srcOpts := minio.CopySrcOptions{
- Bucket: bucketName,
- Object: objectName,
- VersionID: infos[0].VersionID,
- }
- args["src"] = srcOpts
- dstOpts := minio.CopyDestOptions{
- Bucket: bucketName,
- Object: objectName + "-copy",
- }
- args["dst"] = dstOpts
- // Perform the Copy
- if _, err = c.CopyObject(context.Background(), dstOpts, srcOpts); err != nil {
- logError(testName, function, args, startTime, "", "CopyObject failed", err)
- return
- }
- // Destination object
- readerCopy, err := c.GetObject(context.Background(), bucketName, objectName+"-copy", minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- defer readerCopy.Close()
- newestContent, err := ioutil.ReadAll(readerCopy)
- if err != nil {
- logError(testName, function, args, startTime, "", "Reading from GetObject reader failed", err)
- return
- }
- if len(newestContent) == 0 || !bytes.Equal(oldestContent, newestContent) {
- logError(testName, function, args, startTime, "", "Unexpected destination object content", err)
- return
- }
- // Delete all objects and their versions as long as the bucket itself
- if err = cleanupVersionedBucket(bucketName, c); err != nil {
- logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- func testConcurrentCopyObjectWithVersioning() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObject()"
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
- if err != nil {
- logError(testName, function, args, startTime, "", "Make bucket failed", err)
- return
- }
- err = c.EnableVersioning(context.Background(), bucketName)
- if err != nil {
- logError(testName, function, args, startTime, "", "Enable versioning failed", err)
- return
- }
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- var testFiles = []string{"datafile-10-kB"}
- for _, testFile := range testFiles {
- r := getDataReader(testFile)
- buf, err := ioutil.ReadAll(r)
- if err != nil {
- logError(testName, function, args, startTime, "", "unexpected failure", err)
- return
- }
- r.Close()
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- }
- objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
- var infos []minio.ObjectInfo
- for info := range objectsInfo {
- if info.Err != nil {
- logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
- return
- }
- infos = append(infos, info)
- }
- sort.Slice(infos, func(i, j int) bool {
- return infos[i].Size < infos[j].Size
- })
- reader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{VersionID: infos[0].VersionID})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject of the oldest version content failed", err)
- return
- }
- oldestContent, err := ioutil.ReadAll(reader)
- if err != nil {
- logError(testName, function, args, startTime, "", "Reading the oldest object version failed", err)
- return
- }
- // Copy Source
- srcOpts := minio.CopySrcOptions{
- Bucket: bucketName,
- Object: objectName,
- VersionID: infos[0].VersionID,
- }
- args["src"] = srcOpts
- dstOpts := minio.CopyDestOptions{
- Bucket: bucketName,
- Object: objectName + "-copy",
- }
- args["dst"] = dstOpts
- // Perform the Copy concurrently
- const n = 10
- var wg sync.WaitGroup
- wg.Add(n)
- var errs [n]error
- for i := 0; i < n; i++ {
- go func(i int) {
- defer wg.Done()
- _, errs[i] = c.CopyObject(context.Background(), dstOpts, srcOpts)
- }(i)
- }
- wg.Wait()
- for _, err := range errs {
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObject failed", err)
- return
- }
- }
- objectsInfo = c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: false, Prefix: dstOpts.Object})
- infos = []minio.ObjectInfo{}
- for info := range objectsInfo {
- // Destination object
- readerCopy, err := c.GetObject(context.Background(), bucketName, objectName+"-copy", minio.GetObjectOptions{VersionID: info.VersionID})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- defer readerCopy.Close()
- newestContent, err := ioutil.ReadAll(readerCopy)
- if err != nil {
- logError(testName, function, args, startTime, "", "Reading from GetObject reader failed", err)
- return
- }
- if len(newestContent) == 0 || !bytes.Equal(oldestContent, newestContent) {
- logError(testName, function, args, startTime, "", "Unexpected destination object content", err)
- return
- }
- infos = append(infos, info)
- }
- if len(infos) != n {
- logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil)
- return
- }
- // Delete all objects and their versions as long as the bucket itself
- if err = cleanupVersionedBucket(bucketName, c); err != nil {
- logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- func testComposeObjectWithVersioning() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "ComposeObject()"
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
- if err != nil {
- logError(testName, function, args, startTime, "", "Make bucket failed", err)
- return
- }
- err = c.EnableVersioning(context.Background(), bucketName)
- if err != nil {
- logError(testName, function, args, startTime, "", "Enable versioning failed", err)
- return
- }
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- // var testFiles = []string{"datafile-5-MB", "datafile-10-kB"}
- var testFiles = []string{"datafile-5-MB", "datafile-10-kB"}
- var testFilesBytes [][]byte
- for _, testFile := range testFiles {
- r := getDataReader(testFile)
- buf, err := ioutil.ReadAll(r)
- if err != nil {
- logError(testName, function, args, startTime, "", "unexpected failure", err)
- return
- }
- r.Close()
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- testFilesBytes = append(testFilesBytes, buf)
- }
- objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
- var results []minio.ObjectInfo
- for info := range objectsInfo {
- if info.Err != nil {
- logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
- return
- }
- results = append(results, info)
- }
- sort.SliceStable(results, func(i, j int) bool {
- return results[i].Size > results[j].Size
- })
- // Source objects to concatenate. We also specify decryption
- // key for each
- src1 := minio.CopySrcOptions{
- Bucket: bucketName,
- Object: objectName,
- VersionID: results[0].VersionID,
- }
- src2 := minio.CopySrcOptions{
- Bucket: bucketName,
- Object: objectName,
- VersionID: results[1].VersionID,
- }
- dst := minio.CopyDestOptions{
- Bucket: bucketName,
- Object: objectName + "-copy",
- }
- _, err = c.ComposeObject(context.Background(), dst, src1, src2)
- if err != nil {
- logError(testName, function, args, startTime, "", "ComposeObject failed", err)
- return
- }
- // Destination object
- readerCopy, err := c.GetObject(context.Background(), bucketName, objectName+"-copy", minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject of the copy object failed", err)
- return
- }
- defer readerCopy.Close()
- copyContentBytes, err := ioutil.ReadAll(readerCopy)
- if err != nil {
- logError(testName, function, args, startTime, "", "Reading from the copy object reader failed", err)
- return
- }
- var expectedContent []byte
- for _, fileBytes := range testFilesBytes {
- expectedContent = append(expectedContent, fileBytes...)
- }
- if len(copyContentBytes) == 0 || !bytes.Equal(copyContentBytes, expectedContent) {
- logError(testName, function, args, startTime, "", "Unexpected destination object content", err)
- return
- }
- // Delete all objects and their versions as long as the bucket itself
- if err = cleanupVersionedBucket(bucketName, c); err != nil {
- logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- func testRemoveObjectWithVersioning() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "DeleteObject()"
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
- if err != nil {
- logError(testName, function, args, startTime, "", "Make bucket failed", err)
- return
- }
- err = c.EnableVersioning(context.Background(), bucketName)
- if err != nil {
- logError(testName, function, args, startTime, "", "Enable versioning failed", err)
- return
- }
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- _, err = c.PutObject(context.Background(), bucketName, objectName, getDataReader("datafile-10-kB"), int64(dataFileMap["datafile-10-kB"]), minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
- var version minio.ObjectInfo
- for info := range objectsInfo {
- if info.Err != nil {
- logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
- return
- }
- version = info
- break
- }
- err = c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{VersionID: version.VersionID})
- if err != nil {
- logError(testName, function, args, startTime, "", "DeleteObject failed", err)
- return
- }
- objectsInfo = c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
- for range objectsInfo {
- logError(testName, function, args, startTime, "", "Unexpected versioning info, should not have any one ", err)
- return
- }
- err = c.RemoveBucket(context.Background(), bucketName)
- if err != nil {
- logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- func testRemoveObjectsWithVersioning() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "DeleteObjects()"
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
- if err != nil {
- logError(testName, function, args, startTime, "", "Make bucket failed", err)
- return
- }
- err = c.EnableVersioning(context.Background(), bucketName)
- if err != nil {
- logError(testName, function, args, startTime, "", "Enable versioning failed", err)
- return
- }
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- _, err = c.PutObject(context.Background(), bucketName, objectName, getDataReader("datafile-10-kB"), int64(dataFileMap["datafile-10-kB"]), minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- objectsVersions := make(chan minio.ObjectInfo)
- go func() {
- objectsVersionsInfo := c.ListObjects(context.Background(), bucketName,
- minio.ListObjectsOptions{WithVersions: true, Recursive: true})
- for info := range objectsVersionsInfo {
- if info.Err != nil {
- logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
- return
- }
- objectsVersions <- info
- }
- close(objectsVersions)
- }()
- removeErrors := c.RemoveObjects(context.Background(), bucketName, objectsVersions, minio.RemoveObjectsOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "DeleteObjects call failed", err)
- return
- }
- for e := range removeErrors {
- if e.Err != nil {
- logError(testName, function, args, startTime, "", "Single delete operation failed", err)
- return
- }
- }
- objectsVersionsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
- for range objectsVersionsInfo {
- logError(testName, function, args, startTime, "", "Unexpected versioning info, should not have any one ", err)
- return
- }
- err = c.RemoveBucket(context.Background(), bucketName)
- if err != nil {
- logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- func testObjectTaggingWithVersioning() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "{Get,Set,Remove}ObjectTagging()"
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
- if err != nil {
- logError(testName, function, args, startTime, "", "Make bucket failed", err)
- return
- }
- err = c.EnableVersioning(context.Background(), bucketName)
- if err != nil {
- logError(testName, function, args, startTime, "", "Enable versioning failed", err)
- return
- }
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- for _, file := range []string{"datafile-1-b", "datafile-10-kB"} {
- _, err = c.PutObject(context.Background(), bucketName, objectName, getDataReader(file), int64(dataFileMap[file]), minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- }
- versionsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
- var versions []minio.ObjectInfo
- for info := range versionsInfo {
- if info.Err != nil {
- logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
- return
- }
- versions = append(versions, info)
- }
- sort.SliceStable(versions, func(i, j int) bool {
- return versions[i].Size < versions[j].Size
- })
- tagsV1 := map[string]string{"key1": "val1"}
- t1, err := tags.MapToObjectTags(tagsV1)
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObjectTagging (1) failed", err)
- return
- }
- err = c.PutObjectTagging(context.Background(), bucketName, objectName, t1, minio.PutObjectTaggingOptions{VersionID: versions[0].VersionID})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObjectTagging (1) failed", err)
- return
- }
- tagsV2 := map[string]string{"key2": "val2"}
- t2, err := tags.MapToObjectTags(tagsV2)
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObjectTagging (1) failed", err)
- return
- }
- err = c.PutObjectTagging(context.Background(), bucketName, objectName, t2, minio.PutObjectTaggingOptions{VersionID: versions[1].VersionID})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObjectTagging (2) failed", err)
- return
- }
- tagsEqual := func(tags1, tags2 map[string]string) bool {
- for k1, v1 := range tags1 {
- v2, found := tags2[k1]
- if found {
- if v1 != v2 {
- return false
- }
- }
- }
- return true
- }
- gotTagsV1, err := c.GetObjectTagging(context.Background(), bucketName, objectName, minio.GetObjectTaggingOptions{VersionID: versions[0].VersionID})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObjectTagging failed", err)
- return
- }
- if !tagsEqual(t1.ToMap(), gotTagsV1.ToMap()) {
- logError(testName, function, args, startTime, "", "Unexpected tags content (1)", err)
- return
- }
- gotTagsV2, err := c.GetObjectTagging(context.Background(), bucketName, objectName, minio.GetObjectTaggingOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObjectTaggingContext failed", err)
- return
- }
- if !tagsEqual(t2.ToMap(), gotTagsV2.ToMap()) {
- logError(testName, function, args, startTime, "", "Unexpected tags content (2)", err)
- return
- }
- err = c.RemoveObjectTagging(context.Background(), bucketName, objectName, minio.RemoveObjectTaggingOptions{VersionID: versions[0].VersionID})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObjectTagging (2) failed", err)
- return
- }
- emptyTags, err := c.GetObjectTagging(context.Background(), bucketName, objectName,
- minio.GetObjectTaggingOptions{VersionID: versions[0].VersionID})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObjectTagging failed", err)
- return
- }
- if len(emptyTags.ToMap()) != 0 {
- logError(testName, function, args, startTime, "", "Unexpected tags content (2)", err)
- return
- }
- // Delete all objects and their versions as long as the bucket itself
- if err = cleanupVersionedBucket(bucketName, c); err != nil {
- logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test PutObject using a large data to trigger multipart readat
- func testPutObjectWithMetadata() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "PutObject(bucketName, objectName, reader,size, opts)"
- args := map[string]interface{}{
- "bucketName": "",
- "objectName": "",
- "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}",
- }
- if !isFullMode() {
- ignoredLog(testName, function, args, startTime, "Skipping functional tests for short/quick runs").Info()
- return
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "Make bucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- bufSize := dataFileMap["datafile-129-MB"]
- var reader = getDataReader("datafile-129-MB")
- defer reader.Close()
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- // Object custom metadata
- customContentType := "custom/contenttype"
- args["metadata"] = map[string][]string{
- "Content-Type": {customContentType},
- "X-Amz-Meta-CustomKey": {"extra spaces in value"},
- }
- _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{
- ContentType: customContentType})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- // Read the data back
- r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- st, err := r.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat failed", err)
- return
- }
- if st.Size != int64(bufSize) {
- logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err)
- return
- }
- if st.ContentType != customContentType && st.ContentType != "application/octet-stream" {
- logError(testName, function, args, startTime, "", "ContentType does not match, expected "+customContentType+" got "+st.ContentType, err)
- return
- }
- if err := crcMatchesName(r, "datafile-129-MB"); err != nil {
- logError(testName, function, args, startTime, "", "data CRC check failed", err)
- return
- }
- if err := r.Close(); err != nil {
- logError(testName, function, args, startTime, "", "Object Close failed", err)
- return
- }
- if err := r.Close(); err == nil {
- logError(testName, function, args, startTime, "", "Object already closed, should respond with error", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- func testPutObjectWithContentLanguage() {
- // initialize logging params
- objectName := "test-object"
- startTime := time.Now()
- testName := getFuncName()
- function := "PutObject(bucketName, objectName, reader, size, opts)"
- args := map[string]interface{}{
- "bucketName": "",
- "objectName": objectName,
- "size": -1,
- "opts": "",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- data := []byte{}
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(data), int64(0), minio.PutObjectOptions{
- ContentLanguage: "en",
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- objInfo, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject failed", err)
- return
- }
- if objInfo.Metadata.Get("Content-Language") != "en" {
- logError(testName, function, args, startTime, "", "Expected content-language 'en' doesn't match with StatObject return value", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test put object with streaming signature.
- func testPutObjectStreaming() {
- // initialize logging params
- objectName := "test-object"
- startTime := time.Now()
- testName := getFuncName()
- function := "PutObject(bucketName, objectName, reader,size,opts)"
- args := map[string]interface{}{
- "bucketName": "",
- "objectName": objectName,
- "size": -1,
- "opts": "",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Upload an object.
- sizes := []int64{0, 64*1024 - 1, 64 * 1024}
- for _, size := range sizes {
- data := newRandomReader(size, size)
- ui, err := c.PutObject(context.Background(), bucketName, objectName, data, int64(size), minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err)
- return
- }
- if ui.Size != size {
- logError(testName, function, args, startTime, "", "PutObjectStreaming result has unexpected size", nil)
- return
- }
- objInfo, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject failed", err)
- return
- }
- if objInfo.Size != size {
- logError(testName, function, args, startTime, "", "Unexpected size", err)
- return
- }
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test get object seeker from the end, using whence set to '2'.
- func testGetObjectSeekEnd() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "GetObject(bucketName, objectName)"
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Generate 33K of data.
- bufSize := dataFileMap["datafile-33-kB"]
- var reader = getDataReader("datafile-33-kB")
- defer reader.Close()
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- buf, err := ioutil.ReadAll(reader)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAll failed", err)
- return
- }
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- // Read the data back
- r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- st, err := r.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat failed", err)
- return
- }
- if st.Size != int64(bufSize) {
- logError(testName, function, args, startTime, "", "Number of bytes read does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err)
- return
- }
- pos, err := r.Seek(-100, 2)
- if err != nil {
- logError(testName, function, args, startTime, "", "Object Seek failed", err)
- return
- }
- if pos != st.Size-100 {
- logError(testName, function, args, startTime, "", "Incorrect position", err)
- return
- }
- buf2 := make([]byte, 100)
- m, err := readFull(r, buf2)
- if err != nil {
- logError(testName, function, args, startTime, "", "Error reading through readFull", err)
- return
- }
- if m != len(buf2) {
- logError(testName, function, args, startTime, "", "Number of bytes dont match, expected "+string(len(buf2))+" got "+string(m), err)
- return
- }
- hexBuf1 := fmt.Sprintf("%02x", buf[len(buf)-100:])
- hexBuf2 := fmt.Sprintf("%02x", buf2[:m])
- if hexBuf1 != hexBuf2 {
- logError(testName, function, args, startTime, "", "Values at same index dont match", err)
- return
- }
- pos, err = r.Seek(-100, 2)
- if err != nil {
- logError(testName, function, args, startTime, "", "Object Seek failed", err)
- return
- }
- if pos != st.Size-100 {
- logError(testName, function, args, startTime, "", "Incorrect position", err)
- return
- }
- if err = r.Close(); err != nil {
- logError(testName, function, args, startTime, "", "ObjectClose failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test get object reader to not throw error on being closed twice.
- func testGetObjectClosedTwice() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "GetObject(bucketName, objectName)"
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Generate 33K of data.
- bufSize := dataFileMap["datafile-33-kB"]
- var reader = getDataReader("datafile-33-kB")
- defer reader.Close()
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- // Read the data back
- r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- st, err := r.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat failed", err)
- return
- }
- if st.Size != int64(bufSize) {
- logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err)
- return
- }
- if err := crcMatchesName(r, "datafile-33-kB"); err != nil {
- logError(testName, function, args, startTime, "", "data CRC check failed", err)
- return
- }
- if err := r.Close(); err != nil {
- logError(testName, function, args, startTime, "", "Object Close failed", err)
- return
- }
- if err := r.Close(); err == nil {
- logError(testName, function, args, startTime, "", "Already closed object. No error returned", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test RemoveObjects request where context cancels after timeout
- func testRemoveObjectsContext() {
- // Initialize logging params.
- startTime := time.Now()
- testName := getFuncName()
- function := "RemoveObjects(ctx, bucketName, objectsCh)"
- args := map[string]interface{}{
- "bucketName": "",
- }
- // Seed random based on current tie.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Enable tracing, write to stdout.
- // c.TraceOn(os.Stderr)
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Generate put data.
- r := bytes.NewReader(bytes.Repeat([]byte("a"), 8))
- // Multi remove of 20 objects.
- nrObjects := 20
- objectsCh := make(chan minio.ObjectInfo)
- go func() {
- defer close(objectsCh)
- for i := 0; i < nrObjects; i++ {
- objectName := "sample" + strconv.Itoa(i) + ".txt"
- info, err := c.PutObject(context.Background(), bucketName, objectName, r, 8,
- minio.PutObjectOptions{ContentType: "application/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- continue
- }
- objectsCh <- minio.ObjectInfo{
- Key: info.Key,
- VersionID: info.VersionID,
- }
- }
- }()
- // Set context to cancel in 1 nanosecond.
- ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
- args["ctx"] = ctx
- defer cancel()
- // Call RemoveObjects API with short timeout.
- errorCh := c.RemoveObjects(ctx, bucketName, objectsCh, minio.RemoveObjectsOptions{})
- // Check for error.
- select {
- case r := <-errorCh:
- if r.Err == nil {
- logError(testName, function, args, startTime, "", "RemoveObjects should fail on short timeout", err)
- return
- }
- }
- // Set context with longer timeout.
- ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
- args["ctx"] = ctx
- defer cancel()
- // Perform RemoveObjects with the longer timeout. Expect the removals to succeed.
- errorCh = c.RemoveObjects(ctx, bucketName, objectsCh, minio.RemoveObjectsOptions{})
- select {
- case r, more := <-errorCh:
- if more || r.Err != nil {
- logError(testName, function, args, startTime, "", "Unexpected error", r.Err)
- return
- }
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test removing multiple objects with Remove API
- func testRemoveMultipleObjects() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "RemoveObjects(bucketName, objectsCh)"
- args := map[string]interface{}{
- "bucketName": "",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Enable tracing, write to stdout.
- // c.TraceOn(os.Stderr)
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- r := bytes.NewReader(bytes.Repeat([]byte("a"), 8))
- // Multi remove of 1100 objects
- nrObjects := 200
- objectsCh := make(chan minio.ObjectInfo)
- go func() {
- defer close(objectsCh)
- // Upload objects and send them to objectsCh
- for i := 0; i < nrObjects; i++ {
- objectName := "sample" + strconv.Itoa(i) + ".txt"
- info, err := c.PutObject(context.Background(), bucketName, objectName, r, 8,
- minio.PutObjectOptions{ContentType: "application/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- continue
- }
- objectsCh <- minio.ObjectInfo{
- Key: info.Key,
- VersionID: info.VersionID,
- }
- }
- }()
- // Call RemoveObjects API
- errorCh := c.RemoveObjects(context.Background(), bucketName, objectsCh, minio.RemoveObjectsOptions{})
- // Check if errorCh doesn't receive any error
- select {
- case r, more := <-errorCh:
- if more {
- logError(testName, function, args, startTime, "", "Unexpected error", r.Err)
- return
- }
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Tests FPutObject of a big file to trigger multipart
- func testFPutObjectMultipart() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "FPutObject(bucketName, objectName, fileName, opts)"
- args := map[string]interface{}{
- "bucketName": "",
- "objectName": "",
- "fileName": "",
- "opts": "",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload.
- var fileName = getMintDataDirFilePath("datafile-129-MB")
- if fileName == "" {
- // Make a temp file with minPartSize bytes of data.
- file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest")
- if err != nil {
- logError(testName, function, args, startTime, "", "TempFile creation failed", err)
- return
- }
- // Upload 2 parts to utilize all 3 'workers' in multipart and still have a part to upload.
- if _, err = io.Copy(file, getDataReader("datafile-129-MB")); err != nil {
- logError(testName, function, args, startTime, "", "Copy failed", err)
- return
- }
- if err = file.Close(); err != nil {
- logError(testName, function, args, startTime, "", "File Close failed", err)
- return
- }
- fileName = file.Name()
- args["fileName"] = fileName
- }
- totalSize := dataFileMap["datafile-129-MB"]
- // Set base object name
- objectName := bucketName + "FPutObject" + "-standard"
- args["objectName"] = objectName
- objectContentType := "testapplication/octet-stream"
- args["objectContentType"] = objectContentType
- // Perform standard FPutObject with contentType provided (Expecting application/octet-stream)
- _, err = c.FPutObject(context.Background(), bucketName, objectName, fileName, minio.PutObjectOptions{ContentType: objectContentType})
- if err != nil {
- logError(testName, function, args, startTime, "", "FPutObject failed", err)
- return
- }
- r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- objInfo, err := r.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "Unexpected error", err)
- return
- }
- if objInfo.Size != int64(totalSize) {
- logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(totalSize))+" got "+string(objInfo.Size), err)
- return
- }
- if objInfo.ContentType != objectContentType && objInfo.ContentType != "application/octet-stream" {
- logError(testName, function, args, startTime, "", "ContentType doesn't match", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Tests FPutObject with null contentType (default = application/octet-stream)
- func testFPutObject() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "FPutObject(bucketName, objectName, fileName, opts)"
- args := map[string]interface{}{
- "bucketName": "",
- "objectName": "",
- "fileName": "",
- "opts": "",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- location := "us-east-1"
- // Make a new bucket.
- args["bucketName"] = bucketName
- args["location"] = location
- function = "MakeBucket(bucketName, location)"
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: location})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Upload 3 parts worth of data to use all 3 of multiparts 'workers' and have an extra part.
- // Use different data in part for multipart tests to check parts are uploaded in correct order.
- var fName = getMintDataDirFilePath("datafile-129-MB")
- if fName == "" {
- // Make a temp file with minPartSize bytes of data.
- file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest")
- if err != nil {
- logError(testName, function, args, startTime, "", "TempFile creation failed", err)
- return
- }
- // Upload 3 parts to utilize all 3 'workers' in multipart and still have a part to upload.
- if _, err = io.Copy(file, getDataReader("datafile-129-MB")); err != nil {
- logError(testName, function, args, startTime, "", "File copy failed", err)
- return
- }
- // Close the file pro-actively for windows.
- if err = file.Close(); err != nil {
- logError(testName, function, args, startTime, "", "File close failed", err)
- return
- }
- defer os.Remove(file.Name())
- fName = file.Name()
- }
- // Set base object name
- function = "FPutObject(bucketName, objectName, fileName, opts)"
- objectName := bucketName + "FPutObject"
- args["objectName"] = objectName + "-standard"
- args["fileName"] = fName
- args["opts"] = minio.PutObjectOptions{ContentType: "application/octet-stream"}
- // Perform standard FPutObject with contentType provided (Expecting application/octet-stream)
- ui, err := c.FPutObject(context.Background(), bucketName, objectName+"-standard", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "FPutObject failed", err)
- return
- }
- if ui.Size != int64(dataFileMap["datafile-129-MB"]) {
- logError(testName, function, args, startTime, "", "FPutObject returned an unexpected upload size", err)
- return
- }
- // Perform FPutObject with no contentType provided (Expecting application/octet-stream)
- args["objectName"] = objectName + "-Octet"
- _, err = c.FPutObject(context.Background(), bucketName, objectName+"-Octet", fName, minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "File close failed", err)
- return
- }
- srcFile, err := os.Open(fName)
- if err != nil {
- logError(testName, function, args, startTime, "", "File open failed", err)
- return
- }
- defer srcFile.Close()
- // Add extension to temp file name
- tmpFile, err := os.Create(fName + ".gtar")
- if err != nil {
- logError(testName, function, args, startTime, "", "File create failed", err)
- return
- }
- _, err = io.Copy(tmpFile, srcFile)
- if err != nil {
- logError(testName, function, args, startTime, "", "File copy failed", err)
- return
- }
- tmpFile.Close()
- // Perform FPutObject with no contentType provided (Expecting application/x-gtar)
- args["objectName"] = objectName + "-GTar"
- args["opts"] = minio.PutObjectOptions{}
- _, err = c.FPutObject(context.Background(), bucketName, objectName+"-GTar", fName+".gtar", minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "FPutObject failed", err)
- return
- }
- // Check headers
- function = "StatObject(bucketName, objectName, opts)"
- args["objectName"] = objectName + "-standard"
- rStandard, err := c.StatObject(context.Background(), bucketName, objectName+"-standard", minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject failed", err)
- return
- }
- if rStandard.ContentType != "application/octet-stream" {
- logError(testName, function, args, startTime, "", "ContentType does not match, expected application/octet-stream, got "+rStandard.ContentType, err)
- return
- }
- function = "StatObject(bucketName, objectName, opts)"
- args["objectName"] = objectName + "-Octet"
- rOctet, err := c.StatObject(context.Background(), bucketName, objectName+"-Octet", minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject failed", err)
- return
- }
- if rOctet.ContentType != "application/octet-stream" {
- logError(testName, function, args, startTime, "", "ContentType does not match, expected application/octet-stream, got "+rOctet.ContentType, err)
- return
- }
- function = "StatObject(bucketName, objectName, opts)"
- args["objectName"] = objectName + "-GTar"
- rGTar, err := c.StatObject(context.Background(), bucketName, objectName+"-GTar", minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject failed", err)
- return
- }
- if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" {
- logError(testName, function, args, startTime, "", "ContentType does not match, expected application/x-gtar or application/octet-stream, got "+rGTar.ContentType, err)
- return
- }
- os.Remove(fName + ".gtar")
- successLogger(testName, function, args, startTime).Info()
- }
- // Tests FPutObject request when context cancels after timeout
- func testFPutObjectContext() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "FPutObject(bucketName, objectName, fileName, opts)"
- args := map[string]interface{}{
- "bucketName": "",
- "objectName": "",
- "fileName": "",
- "opts": "",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Upload 1 parts worth of data to use multipart upload.
- // Use different data in part for multipart tests to check parts are uploaded in correct order.
- var fName = getMintDataDirFilePath("datafile-1-MB")
- if fName == "" {
- // Make a temp file with 1 MiB bytes of data.
- file, err := ioutil.TempFile(os.TempDir(), "FPutObjectContextTest")
- if err != nil {
- logError(testName, function, args, startTime, "", "TempFile creation failed", err)
- return
- }
- // Upload 1 parts to trigger multipart upload
- if _, err = io.Copy(file, getDataReader("datafile-1-MB")); err != nil {
- logError(testName, function, args, startTime, "", "File copy failed", err)
- return
- }
- // Close the file pro-actively for windows.
- if err = file.Close(); err != nil {
- logError(testName, function, args, startTime, "", "File close failed", err)
- return
- }
- defer os.Remove(file.Name())
- fName = file.Name()
- }
- // Set base object name
- objectName := bucketName + "FPutObjectContext"
- args["objectName"] = objectName
- ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
- args["ctx"] = ctx
- defer cancel()
- // Perform FPutObject with contentType provided (Expecting application/octet-stream)
- _, err = c.FPutObject(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"})
- if err == nil {
- logError(testName, function, args, startTime, "", "FPutObject should fail on short timeout", err)
- return
- }
- ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
- defer cancel()
- // Perform FPutObject with a long timeout. Expect the put object to succeed
- _, err = c.FPutObject(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "FPutObject shouldn't fail on long timeout", err)
- return
- }
- _, err = c.StatObject(context.Background(), bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Tests FPutObject request when context cancels after timeout
- func testFPutObjectContextV2() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "FPutObjectContext(ctx, bucketName, objectName, fileName, opts)"
- args := map[string]interface{}{
- "bucketName": "",
- "objectName": "",
- "opts": "minio.PutObjectOptions{ContentType:objectContentType}",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Upload 1 parts worth of data to use multipart upload.
- // Use different data in part for multipart tests to check parts are uploaded in correct order.
- var fName = getMintDataDirFilePath("datafile-1-MB")
- if fName == "" {
- // Make a temp file with 1 MiB bytes of data.
- file, err := ioutil.TempFile(os.TempDir(), "FPutObjectContextTest")
- if err != nil {
- logError(testName, function, args, startTime, "", "Temp file creation failed", err)
- return
- }
- // Upload 1 parts to trigger multipart upload
- if _, err = io.Copy(file, getDataReader("datafile-1-MB")); err != nil {
- logError(testName, function, args, startTime, "", "File copy failed", err)
- return
- }
- // Close the file pro-actively for windows.
- if err = file.Close(); err != nil {
- logError(testName, function, args, startTime, "", "File close failed", err)
- return
- }
- defer os.Remove(file.Name())
- fName = file.Name()
- }
- // Set base object name
- objectName := bucketName + "FPutObjectContext"
- args["objectName"] = objectName
- ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
- args["ctx"] = ctx
- defer cancel()
- // Perform FPutObject with contentType provided (Expecting application/octet-stream)
- _, err = c.FPutObject(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"})
- if err == nil {
- logError(testName, function, args, startTime, "", "FPutObject should fail on short timeout", err)
- return
- }
- ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
- defer cancel()
- // Perform FPutObject with a long timeout. Expect the put object to succeed
- _, err = c.FPutObject(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "FPutObject shouldn't fail on longer timeout", err)
- return
- }
- _, err = c.StatObject(context.Background(), bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test validates putObject with context to see if request cancellation is honored.
- func testPutObjectContext() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "PutObject(ctx, bucketName, objectName, fileName, opts)"
- args := map[string]interface{}{
- "ctx": "",
- "bucketName": "",
- "objectName": "",
- "opts": "",
- }
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Make a new bucket.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket call failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- bufSize := dataFileMap["datafile-33-kB"]
- var reader = getDataReader("datafile-33-kB")
- defer reader.Close()
- objectName := fmt.Sprintf("test-file-%v", rand.Uint32())
- args["objectName"] = objectName
- ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
- cancel()
- args["ctx"] = ctx
- args["opts"] = minio.PutObjectOptions{ContentType: "binary/octet-stream"}
- _, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err == nil {
- logError(testName, function, args, startTime, "", "PutObject should fail on short timeout", err)
- return
- }
- ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
- args["ctx"] = ctx
- defer cancel()
- reader = getDataReader("datafile-33-kB")
- defer reader.Close()
- _, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject with long timeout failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Tests get object ReaderSeeker interface methods.
- func testGetObjectReadSeekFunctional() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "GetObject(bucketName, objectName)"
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer func() {
- // Delete all objects and buckets
- if err = cleanupBucket(bucketName, c); err != nil {
- logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
- return
- }
- }()
- // Generate 33K of data.
- bufSize := dataFileMap["datafile-33-kB"]
- var reader = getDataReader("datafile-33-kB")
- defer reader.Close()
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- buf, err := ioutil.ReadAll(reader)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAll failed", err)
- return
- }
- // Save the data
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- // Read the data back
- r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- st, err := r.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat object failed", err)
- return
- }
- if st.Size != int64(bufSize) {
- logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err)
- return
- }
- // This following function helps us to compare data from the reader after seek
- // with the data from the original buffer
- cmpData := func(r io.Reader, start, end int) {
- if end-start == 0 {
- return
- }
- buffer := bytes.NewBuffer([]byte{})
- if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil {
- if err != io.EOF {
- logError(testName, function, args, startTime, "", "CopyN failed", err)
- return
- }
- }
- if !bytes.Equal(buf[start:end], buffer.Bytes()) {
- logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err)
- return
- }
- }
- // Generic seek error for errors other than io.EOF
- seekErr := errors.New("seek error")
- testCases := []struct {
- offset int64
- whence int
- pos int64
- err error
- shouldCmp bool
- start int
- end int
- }{
- // Start from offset 0, fetch data and compare
- {0, 0, 0, nil, true, 0, 0},
- // Start from offset 2048, fetch data and compare
- {2048, 0, 2048, nil, true, 2048, bufSize},
- // Start from offset larger than possible
- {int64(bufSize) + 1024, 0, 0, seekErr, false, 0, 0},
- // Move to offset 0 without comparing
- {0, 0, 0, nil, false, 0, 0},
- // Move one step forward and compare
- {1, 1, 1, nil, true, 1, bufSize},
- // Move larger than possible
- {int64(bufSize), 1, 0, seekErr, false, 0, 0},
- // Provide negative offset with CUR_SEEK
- {int64(-1), 1, 0, seekErr, false, 0, 0},
- // Test with whence SEEK_END and with positive offset
- {1024, 2, int64(bufSize) - 1024, io.EOF, true, 0, 0},
- // Test with whence SEEK_END and with negative offset
- {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize},
- // Test with whence SEEK_END and with large negative offset
- {-int64(bufSize) * 2, 2, 0, seekErr, true, 0, 0},
- }
- for i, testCase := range testCases {
- // Perform seek operation
- n, err := r.Seek(testCase.offset, testCase.whence)
- // We expect an error
- if testCase.err == seekErr && err == nil {
- logError(testName, function, args, startTime, "", "Test "+string(i+1)+", unexpected err value: expected: "+testCase.err.Error()+", found: "+err.Error(), err)
- return
- }
- // We expect a specific error
- if testCase.err != seekErr && testCase.err != err {
- logError(testName, function, args, startTime, "", "Test "+string(i+1)+", unexpected err value: expected: "+testCase.err.Error()+", found: "+err.Error(), err)
- return
- }
- // If we expect an error go to the next loop
- if testCase.err != nil {
- continue
- }
- // Check the returned seek pos
- if n != testCase.pos {
- logError(testName, function, args, startTime, "", "Test "+string(i+1)+", number of bytes seeked does not match, expected "+string(testCase.pos)+", got "+string(n), err)
- return
- }
- // Compare only if shouldCmp is activated
- if testCase.shouldCmp {
- cmpData(r, testCase.start, testCase.end)
- }
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Tests get object ReaderAt interface methods.
- func testGetObjectReadAtFunctional() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "GetObject(bucketName, objectName)"
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Generate 33K of data.
- bufSize := dataFileMap["datafile-33-kB"]
- var reader = getDataReader("datafile-33-kB")
- defer reader.Close()
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- buf, err := ioutil.ReadAll(reader)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAll failed", err)
- return
- }
- // Save the data
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- // read the data back
- r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- offset := int64(2048)
- // read directly
- buf1 := make([]byte, 512)
- buf2 := make([]byte, 512)
- buf3 := make([]byte, 512)
- buf4 := make([]byte, 512)
- // Test readAt before stat is called such that objectInfo doesn't change.
- m, err := r.ReadAt(buf1, offset)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- if m != len(buf1) {
- logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err)
- return
- }
- if !bytes.Equal(buf1, buf[offset:offset+512]) {
- logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
- return
- }
- offset += 512
- st, err := r.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat failed", err)
- return
- }
- if st.Size != int64(bufSize) {
- logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err)
- return
- }
- m, err = r.ReadAt(buf2, offset)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- if m != len(buf2) {
- logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err)
- return
- }
- if !bytes.Equal(buf2, buf[offset:offset+512]) {
- logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
- return
- }
- offset += 512
- m, err = r.ReadAt(buf3, offset)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- if m != len(buf3) {
- logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err)
- return
- }
- if !bytes.Equal(buf3, buf[offset:offset+512]) {
- logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
- return
- }
- offset += 512
- m, err = r.ReadAt(buf4, offset)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- if m != len(buf4) {
- logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err)
- return
- }
- if !bytes.Equal(buf4, buf[offset:offset+512]) {
- logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
- return
- }
- buf5 := make([]byte, len(buf))
- // Read the whole object.
- m, err = r.ReadAt(buf5, 0)
- if err != nil {
- if err != io.EOF {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- }
- if m != len(buf5) {
- logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err)
- return
- }
- if !bytes.Equal(buf, buf5) {
- logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err)
- return
- }
- buf6 := make([]byte, len(buf)+1)
- // Read the whole object and beyond.
- _, err = r.ReadAt(buf6, 0)
- if err != nil {
- if err != io.EOF {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Reproduces issue https://github.com/minio/minio-go/issues/1137
- func testGetObjectReadAtWhenEOFWasReached() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "GetObject(bucketName, objectName)"
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Generate 33K of data.
- bufSize := dataFileMap["datafile-33-kB"]
- var reader = getDataReader("datafile-33-kB")
- defer reader.Close()
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- buf, err := ioutil.ReadAll(reader)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAll failed", err)
- return
- }
- // Save the data
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- // read the data back
- r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- // read directly
- buf1 := make([]byte, len(buf))
- buf2 := make([]byte, 512)
- m, err := r.Read(buf1)
- if err != nil {
- if err != io.EOF {
- logError(testName, function, args, startTime, "", "Read failed", err)
- return
- }
- }
- if m != len(buf1) {
- logError(testName, function, args, startTime, "", "Read read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err)
- return
- }
- if !bytes.Equal(buf1, buf) {
- logError(testName, function, args, startTime, "", "Incorrect count of Read data", err)
- return
- }
- st, err := r.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat failed", err)
- return
- }
- if st.Size != int64(bufSize) {
- logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err)
- return
- }
- m, err = r.ReadAt(buf2, 512)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- if m != len(buf2) {
- logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err)
- return
- }
- if !bytes.Equal(buf2, buf[512:1024]) {
- logError(testName, function, args, startTime, "", "Incorrect count of ReadAt data", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test Presigned Post Policy
- func testPresignedPostPolicy() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "PresignedPostPolicy(policy)"
- args := map[string]interface{}{
- "policy": "",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- // Make a new bucket in 'us-east-1' (source bucket).
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Generate 33K of data.
- var reader = getDataReader("datafile-33-kB")
- defer reader.Close()
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- // Azure requires the key to not start with a number
- metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "user")
- metadataValue := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- buf, err := ioutil.ReadAll(reader)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAll failed", err)
- return
- }
- // Save the data
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- policy := minio.NewPostPolicy()
- if err := policy.SetBucket(""); err == nil {
- logError(testName, function, args, startTime, "", "SetBucket did not fail for invalid conditions", err)
- return
- }
- if err := policy.SetKey(""); err == nil {
- logError(testName, function, args, startTime, "", "SetKey did not fail for invalid conditions", err)
- return
- }
- if err := policy.SetKeyStartsWith(""); err == nil {
- logError(testName, function, args, startTime, "", "SetKeyStartsWith did not fail for invalid conditions", err)
- return
- }
- if err := policy.SetExpires(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)); err == nil {
- logError(testName, function, args, startTime, "", "SetExpires did not fail for invalid conditions", err)
- return
- }
- if err := policy.SetContentType(""); err == nil {
- logError(testName, function, args, startTime, "", "SetContentType did not fail for invalid conditions", err)
- return
- }
- if err := policy.SetContentTypeStartsWith(""); err == nil {
- logError(testName, function, args, startTime, "", "SetContentTypeStartsWith did not fail for invalid conditions", err)
- return
- }
- if err := policy.SetContentLengthRange(1024*1024, 1024); err == nil {
- logError(testName, function, args, startTime, "", "SetContentLengthRange did not fail for invalid conditions", err)
- return
- }
- if err := policy.SetUserMetadata("", ""); err == nil {
- logError(testName, function, args, startTime, "", "SetUserMetadata did not fail for invalid conditions", err)
- return
- }
- policy.SetBucket(bucketName)
- policy.SetKey(objectName)
- policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days
- policy.SetContentType("binary/octet-stream")
- policy.SetContentLengthRange(10, 1024*1024)
- policy.SetUserMetadata(metadataKey, metadataValue)
- args["policy"] = policy.String()
- presignedPostPolicyURL, formData, err := c.PresignedPostPolicy(context.Background(), policy)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedPostPolicy failed", err)
- return
- }
- var formBuf bytes.Buffer
- writer := multipart.NewWriter(&formBuf)
- for k, v := range formData {
- writer.WriteField(k, v)
- }
- // Get a 33KB file to upload and test if set post policy works
- var filePath = getMintDataDirFilePath("datafile-33-kB")
- if filePath == "" {
- // Make a temp file with 33 KB data.
- file, err := ioutil.TempFile(os.TempDir(), "PresignedPostPolicyTest")
- if err != nil {
- logError(testName, function, args, startTime, "", "TempFile creation failed", err)
- return
- }
- if _, err = io.Copy(file, getDataReader("datafile-33-kB")); err != nil {
- logError(testName, function, args, startTime, "", "Copy failed", err)
- return
- }
- if err = file.Close(); err != nil {
- logError(testName, function, args, startTime, "", "File Close failed", err)
- return
- }
- filePath = file.Name()
- }
- // add file to post request
- f, err := os.Open(filePath)
- defer f.Close()
- if err != nil {
- logError(testName, function, args, startTime, "", "File open failed", err)
- return
- }
- w, err := writer.CreateFormFile("file", filePath)
- if err != nil {
- logError(testName, function, args, startTime, "", "CreateFormFile failed", err)
- return
- }
- _, err = io.Copy(w, f)
- if err != nil {
- logError(testName, function, args, startTime, "", "Copy failed", err)
- return
- }
- writer.Close()
- transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS)))
- if err != nil {
- logError(testName, function, args, startTime, "", "DefaultTransport failed", err)
- return
- }
- httpClient := &http.Client{
- // Setting a sensible time out of 30secs to wait for response
- // headers. Request is pro-actively canceled after 30secs
- // with no response.
- Timeout: 30 * time.Second,
- Transport: transport,
- }
- req, err := http.NewRequest(http.MethodPost, presignedPostPolicyURL.String(), bytes.NewReader(formBuf.Bytes()))
- if err != nil {
- logError(testName, function, args, startTime, "", "Http request failed", err)
- return
- }
- req.Header.Set("Content-Type", writer.FormDataContentType())
- // make post request with correct form data
- res, err := httpClient.Do(req)
- if err != nil {
- logError(testName, function, args, startTime, "", "Http request failed", err)
- return
- }
- defer res.Body.Close()
- if res.StatusCode != http.StatusNoContent {
- logError(testName, function, args, startTime, "", "Http request failed", errors.New(res.Status))
- return
- }
- // expected path should be absolute path of the object
- var scheme string
- if mustParseBool(os.Getenv(enableHTTPS)) {
- scheme = "https://"
- } else {
- scheme = "http://"
- }
- expectedLocation := scheme + os.Getenv(serverEndpoint) + "/" + bucketName + "/" + objectName
- expectedLocationBucketDNS := scheme + bucketName + "." + os.Getenv(serverEndpoint) + "/" + objectName
- if val, ok := res.Header["Location"]; ok {
- if val[0] != expectedLocation && val[0] != expectedLocationBucketDNS {
- logError(testName, function, args, startTime, "", "Location in header response is incorrect", err)
- return
- }
- } else {
- logError(testName, function, args, startTime, "", "Location not found in header response", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Tests copy object
- func testCopyObject() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObject(dst, src)"
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- // Make a new bucket in 'us-east-1' (source bucket).
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Make a new bucket in 'us-east-1' (destination bucket).
- err = c.MakeBucket(context.Background(), bucketName+"-copy", minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName+"-copy", c)
- // Generate 33K of data.
- bufSize := dataFileMap["datafile-33-kB"]
- var reader = getDataReader("datafile-33-kB")
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- // Check the various fields of source object against destination object.
- objInfo, err := r.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat failed", err)
- return
- }
- // Copy Source
- src := minio.CopySrcOptions{
- Bucket: bucketName,
- Object: objectName,
- // Set copy conditions.
- MatchETag: objInfo.ETag,
- MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC),
- }
- args["src"] = src
- dst := minio.CopyDestOptions{
- Bucket: bucketName + "-copy",
- Object: objectName + "-copy",
- }
- // Perform the Copy
- if _, err = c.CopyObject(context.Background(), dst, src); err != nil {
- logError(testName, function, args, startTime, "", "CopyObject failed", err)
- return
- }
- // Source object
- r, err = c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- // Destination object
- readerCopy, err := c.GetObject(context.Background(), bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- // Check the various fields of source object against destination object.
- objInfo, err = r.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat failed", err)
- return
- }
- objInfoCopy, err := readerCopy.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat failed", err)
- return
- }
- if objInfo.Size != objInfoCopy.Size {
- logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(objInfoCopy.Size)+", got "+string(objInfo.Size), err)
- return
- }
- if err := crcMatchesName(r, "datafile-33-kB"); err != nil {
- logError(testName, function, args, startTime, "", "data CRC check failed", err)
- return
- }
- if err := crcMatchesName(readerCopy, "datafile-33-kB"); err != nil {
- logError(testName, function, args, startTime, "", "copy data CRC check failed", err)
- return
- }
- // Close all the get readers before proceeding with CopyObject operations.
- r.Close()
- readerCopy.Close()
- // CopyObject again but with wrong conditions
- src = minio.CopySrcOptions{
- Bucket: bucketName,
- Object: objectName,
- MatchUnmodifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC),
- NoMatchETag: objInfo.ETag,
- }
- // Perform the Copy which should fail
- _, err = c.CopyObject(context.Background(), dst, src)
- if err == nil {
- logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err)
- return
- }
- src = minio.CopySrcOptions{
- Bucket: bucketName,
- Object: objectName,
- }
- dst = minio.CopyDestOptions{
- Bucket: bucketName,
- Object: objectName,
- ReplaceMetadata: true,
- UserMetadata: map[string]string{
- "Copy": "should be same",
- },
- }
- args["dst"] = dst
- args["src"] = src
- _, err = c.CopyObject(context.Background(), dst, src)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObject shouldn't fail", err)
- return
- }
- oi, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject failed", err)
- return
- }
- stOpts := minio.StatObjectOptions{}
- stOpts.SetMatchETag(oi.ETag)
- objInfo, err = c.StatObject(context.Background(), bucketName, objectName, stOpts)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObject ETag should match and not fail", err)
- return
- }
- if objInfo.Metadata.Get("x-amz-meta-copy") != "should be same" {
- logError(testName, function, args, startTime, "", "CopyObject modified metadata should match", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Tests SSE-C get object ReaderSeeker interface methods.
- func testSSECEncryptedGetObjectReadSeekFunctional() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "GetObject(bucketName, objectName)"
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer func() {
- // Delete all objects and buckets
- if err = cleanupBucket(bucketName, c); err != nil {
- logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
- return
- }
- }()
- // Generate 129MiB of data.
- bufSize := dataFileMap["datafile-129-MB"]
- var reader = getDataReader("datafile-129-MB")
- defer reader.Close()
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- buf, err := ioutil.ReadAll(reader)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAll failed", err)
- return
- }
- // Save the data
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{
- ContentType: "binary/octet-stream",
- ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- // Read the data back
- r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{
- ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- defer r.Close()
- st, err := r.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat object failed", err)
- return
- }
- if st.Size != int64(bufSize) {
- logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err)
- return
- }
- // This following function helps us to compare data from the reader after seek
- // with the data from the original buffer
- cmpData := func(r io.Reader, start, end int) {
- if end-start == 0 {
- return
- }
- buffer := bytes.NewBuffer([]byte{})
- if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil {
- if err != io.EOF {
- logError(testName, function, args, startTime, "", "CopyN failed", err)
- return
- }
- }
- if !bytes.Equal(buf[start:end], buffer.Bytes()) {
- logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err)
- return
- }
- }
- testCases := []struct {
- offset int64
- whence int
- pos int64
- err error
- shouldCmp bool
- start int
- end int
- }{
- // Start from offset 0, fetch data and compare
- {0, 0, 0, nil, true, 0, 0},
- // Start from offset 2048, fetch data and compare
- {2048, 0, 2048, nil, true, 2048, bufSize},
- // Start from offset larger than possible
- {int64(bufSize) + 1024, 0, 0, io.EOF, false, 0, 0},
- // Move to offset 0 without comparing
- {0, 0, 0, nil, false, 0, 0},
- // Move one step forward and compare
- {1, 1, 1, nil, true, 1, bufSize},
- // Move larger than possible
- {int64(bufSize), 1, 0, io.EOF, false, 0, 0},
- // Provide negative offset with CUR_SEEK
- {int64(-1), 1, 0, fmt.Errorf("Negative position not allowed for 1"), false, 0, 0},
- // Test with whence SEEK_END and with positive offset
- {1024, 2, 0, io.EOF, false, 0, 0},
- // Test with whence SEEK_END and with negative offset
- {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize},
- // Test with whence SEEK_END and with large negative offset
- {-int64(bufSize) * 2, 2, 0, fmt.Errorf("Seeking at negative offset not allowed for 2"), false, 0, 0},
- // Test with invalid whence
- {0, 3, 0, fmt.Errorf("Invalid whence 3"), false, 0, 0},
- }
- for i, testCase := range testCases {
- // Perform seek operation
- n, err := r.Seek(testCase.offset, testCase.whence)
- if err != nil && testCase.err == nil {
- // We expected success.
- logError(testName, function, args, startTime, "",
- fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err)
- return
- }
- if err == nil && testCase.err != nil {
- // We expected failure, but got success.
- logError(testName, function, args, startTime, "",
- fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err)
- return
- }
- if err != nil && testCase.err != nil {
- if err.Error() != testCase.err.Error() {
- // We expect a specific error
- logError(testName, function, args, startTime, "",
- fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err)
- return
- }
- }
- // Check the returned seek pos
- if n != testCase.pos {
- logError(testName, function, args, startTime, "",
- fmt.Sprintf("Test %d, number of bytes seeked does not match, expected %d, got %d", i+1, testCase.pos, n), err)
- return
- }
- // Compare only if shouldCmp is activated
- if testCase.shouldCmp {
- cmpData(r, testCase.start, testCase.end)
- }
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Tests SSE-S3 get object ReaderSeeker interface methods.
- func testSSES3EncryptedGetObjectReadSeekFunctional() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "GetObject(bucketName, objectName)"
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer func() {
- // Delete all objects and buckets
- if err = cleanupBucket(bucketName, c); err != nil {
- logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
- return
- }
- }()
- // Generate 129MiB of data.
- bufSize := dataFileMap["datafile-129-MB"]
- var reader = getDataReader("datafile-129-MB")
- defer reader.Close()
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- buf, err := ioutil.ReadAll(reader)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAll failed", err)
- return
- }
- // Save the data
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{
- ContentType: "binary/octet-stream",
- ServerSideEncryption: encrypt.NewSSE(),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- // Read the data back
- r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- defer r.Close()
- st, err := r.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat object failed", err)
- return
- }
- if st.Size != int64(bufSize) {
- logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err)
- return
- }
- // This following function helps us to compare data from the reader after seek
- // with the data from the original buffer
- cmpData := func(r io.Reader, start, end int) {
- if end-start == 0 {
- return
- }
- buffer := bytes.NewBuffer([]byte{})
- if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil {
- if err != io.EOF {
- logError(testName, function, args, startTime, "", "CopyN failed", err)
- return
- }
- }
- if !bytes.Equal(buf[start:end], buffer.Bytes()) {
- logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err)
- return
- }
- }
- testCases := []struct {
- offset int64
- whence int
- pos int64
- err error
- shouldCmp bool
- start int
- end int
- }{
- // Start from offset 0, fetch data and compare
- {0, 0, 0, nil, true, 0, 0},
- // Start from offset 2048, fetch data and compare
- {2048, 0, 2048, nil, true, 2048, bufSize},
- // Start from offset larger than possible
- {int64(bufSize) + 1024, 0, 0, io.EOF, false, 0, 0},
- // Move to offset 0 without comparing
- {0, 0, 0, nil, false, 0, 0},
- // Move one step forward and compare
- {1, 1, 1, nil, true, 1, bufSize},
- // Move larger than possible
- {int64(bufSize), 1, 0, io.EOF, false, 0, 0},
- // Provide negative offset with CUR_SEEK
- {int64(-1), 1, 0, fmt.Errorf("Negative position not allowed for 1"), false, 0, 0},
- // Test with whence SEEK_END and with positive offset
- {1024, 2, 0, io.EOF, false, 0, 0},
- // Test with whence SEEK_END and with negative offset
- {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize},
- // Test with whence SEEK_END and with large negative offset
- {-int64(bufSize) * 2, 2, 0, fmt.Errorf("Seeking at negative offset not allowed for 2"), false, 0, 0},
- // Test with invalid whence
- {0, 3, 0, fmt.Errorf("Invalid whence 3"), false, 0, 0},
- }
- for i, testCase := range testCases {
- // Perform seek operation
- n, err := r.Seek(testCase.offset, testCase.whence)
- if err != nil && testCase.err == nil {
- // We expected success.
- logError(testName, function, args, startTime, "",
- fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err)
- return
- }
- if err == nil && testCase.err != nil {
- // We expected failure, but got success.
- logError(testName, function, args, startTime, "",
- fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err)
- return
- }
- if err != nil && testCase.err != nil {
- if err.Error() != testCase.err.Error() {
- // We expect a specific error
- logError(testName, function, args, startTime, "",
- fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err)
- return
- }
- }
- // Check the returned seek pos
- if n != testCase.pos {
- logError(testName, function, args, startTime, "",
- fmt.Sprintf("Test %d, number of bytes seeked does not match, expected %d, got %d", i+1, testCase.pos, n), err)
- return
- }
- // Compare only if shouldCmp is activated
- if testCase.shouldCmp {
- cmpData(r, testCase.start, testCase.end)
- }
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Tests SSE-C get object ReaderAt interface methods.
- func testSSECEncryptedGetObjectReadAtFunctional() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "GetObject(bucketName, objectName)"
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Generate 129MiB of data.
- bufSize := dataFileMap["datafile-129-MB"]
- var reader = getDataReader("datafile-129-MB")
- defer reader.Close()
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- buf, err := ioutil.ReadAll(reader)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAll failed", err)
- return
- }
- // Save the data
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{
- ContentType: "binary/octet-stream",
- ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- // read the data back
- r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{
- ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- defer r.Close()
- offset := int64(2048)
- // read directly
- buf1 := make([]byte, 512)
- buf2 := make([]byte, 512)
- buf3 := make([]byte, 512)
- buf4 := make([]byte, 512)
- // Test readAt before stat is called such that objectInfo doesn't change.
- m, err := r.ReadAt(buf1, offset)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- if m != len(buf1) {
- logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err)
- return
- }
- if !bytes.Equal(buf1, buf[offset:offset+512]) {
- logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
- return
- }
- offset += 512
- st, err := r.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat failed", err)
- return
- }
- if st.Size != int64(bufSize) {
- logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err)
- return
- }
- m, err = r.ReadAt(buf2, offset)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- if m != len(buf2) {
- logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err)
- return
- }
- if !bytes.Equal(buf2, buf[offset:offset+512]) {
- logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
- return
- }
- offset += 512
- m, err = r.ReadAt(buf3, offset)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- if m != len(buf3) {
- logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err)
- return
- }
- if !bytes.Equal(buf3, buf[offset:offset+512]) {
- logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
- return
- }
- offset += 512
- m, err = r.ReadAt(buf4, offset)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- if m != len(buf4) {
- logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err)
- return
- }
- if !bytes.Equal(buf4, buf[offset:offset+512]) {
- logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
- return
- }
- buf5 := make([]byte, len(buf))
- // Read the whole object.
- m, err = r.ReadAt(buf5, 0)
- if err != nil {
- if err != io.EOF {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- }
- if m != len(buf5) {
- logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err)
- return
- }
- if !bytes.Equal(buf, buf5) {
- logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err)
- return
- }
- buf6 := make([]byte, len(buf)+1)
- // Read the whole object and beyond.
- _, err = r.ReadAt(buf6, 0)
- if err != nil {
- if err != io.EOF {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Tests SSE-S3 get object ReaderAt interface methods.
- func testSSES3EncryptedGetObjectReadAtFunctional() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "GetObject(bucketName, objectName)"
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Generate 129MiB of data.
- bufSize := dataFileMap["datafile-129-MB"]
- var reader = getDataReader("datafile-129-MB")
- defer reader.Close()
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- buf, err := ioutil.ReadAll(reader)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAll failed", err)
- return
- }
- // Save the data
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{
- ContentType: "binary/octet-stream",
- ServerSideEncryption: encrypt.NewSSE(),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- // read the data back
- r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- defer r.Close()
- offset := int64(2048)
- // read directly
- buf1 := make([]byte, 512)
- buf2 := make([]byte, 512)
- buf3 := make([]byte, 512)
- buf4 := make([]byte, 512)
- // Test readAt before stat is called such that objectInfo doesn't change.
- m, err := r.ReadAt(buf1, offset)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- if m != len(buf1) {
- logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err)
- return
- }
- if !bytes.Equal(buf1, buf[offset:offset+512]) {
- logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
- return
- }
- offset += 512
- st, err := r.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat failed", err)
- return
- }
- if st.Size != int64(bufSize) {
- logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err)
- return
- }
- m, err = r.ReadAt(buf2, offset)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- if m != len(buf2) {
- logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err)
- return
- }
- if !bytes.Equal(buf2, buf[offset:offset+512]) {
- logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
- return
- }
- offset += 512
- m, err = r.ReadAt(buf3, offset)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- if m != len(buf3) {
- logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err)
- return
- }
- if !bytes.Equal(buf3, buf[offset:offset+512]) {
- logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
- return
- }
- offset += 512
- m, err = r.ReadAt(buf4, offset)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- if m != len(buf4) {
- logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err)
- return
- }
- if !bytes.Equal(buf4, buf[offset:offset+512]) {
- logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
- return
- }
- buf5 := make([]byte, len(buf))
- // Read the whole object.
- m, err = r.ReadAt(buf5, 0)
- if err != nil {
- if err != io.EOF {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- }
- if m != len(buf5) {
- logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err)
- return
- }
- if !bytes.Equal(buf, buf5) {
- logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err)
- return
- }
- buf6 := make([]byte, len(buf)+1)
- // Read the whole object and beyond.
- _, err = r.ReadAt(buf6, 0)
- if err != nil {
- if err != io.EOF {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // testSSECEncryptionPutGet tests encryption with customer provided encryption keys
- func testSSECEncryptionPutGet() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "PutEncryptedObject(bucketName, objectName, reader, sse)"
- args := map[string]interface{}{
- "bucketName": "",
- "objectName": "",
- "sse": "",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- testCases := []struct {
- buf []byte
- }{
- {buf: bytes.Repeat([]byte("F"), 1)},
- {buf: bytes.Repeat([]byte("F"), 15)},
- {buf: bytes.Repeat([]byte("F"), 16)},
- {buf: bytes.Repeat([]byte("F"), 17)},
- {buf: bytes.Repeat([]byte("F"), 31)},
- {buf: bytes.Repeat([]byte("F"), 32)},
- {buf: bytes.Repeat([]byte("F"), 33)},
- {buf: bytes.Repeat([]byte("F"), 1024)},
- {buf: bytes.Repeat([]byte("F"), 1024*2)},
- {buf: bytes.Repeat([]byte("F"), 1024*1024)},
- }
- const password = "correct horse battery staple" // https://xkcd.com/936/
- for i, testCase := range testCases {
- // Generate a random object name
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- // Secured object
- sse := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName))
- args["sse"] = sse
- // Put encrypted data
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(testCase.buf), int64(len(testCase.buf)), minio.PutObjectOptions{ServerSideEncryption: sse})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutEncryptedObject failed", err)
- return
- }
- // Read the data back
- r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err)
- return
- }
- defer r.Close()
- // Compare the sent object with the received one
- recvBuffer := bytes.NewBuffer([]byte{})
- if _, err = io.Copy(recvBuffer, r); err != nil {
- logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err)
- return
- }
- if recvBuffer.Len() != len(testCase.buf) {
- logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err)
- return
- }
- if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) {
- logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // TestEncryptionFPut tests encryption with customer specified encryption keys
- func testSSECEncryptionFPut() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "FPutEncryptedObject(bucketName, objectName, filePath, contentType, sse)"
- args := map[string]interface{}{
- "bucketName": "",
- "objectName": "",
- "filePath": "",
- "contentType": "",
- "sse": "",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Object custom metadata
- customContentType := "custom/contenttype"
- args["metadata"] = customContentType
- testCases := []struct {
- buf []byte
- }{
- {buf: bytes.Repeat([]byte("F"), 0)},
- {buf: bytes.Repeat([]byte("F"), 1)},
- {buf: bytes.Repeat([]byte("F"), 15)},
- {buf: bytes.Repeat([]byte("F"), 16)},
- {buf: bytes.Repeat([]byte("F"), 17)},
- {buf: bytes.Repeat([]byte("F"), 31)},
- {buf: bytes.Repeat([]byte("F"), 32)},
- {buf: bytes.Repeat([]byte("F"), 33)},
- {buf: bytes.Repeat([]byte("F"), 1024)},
- {buf: bytes.Repeat([]byte("F"), 1024*2)},
- {buf: bytes.Repeat([]byte("F"), 1024*1024)},
- }
- const password = "correct horse battery staple" // https://xkcd.com/936/
- for i, testCase := range testCases {
- // Generate a random object name
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- // Secured object
- sse := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName))
- args["sse"] = sse
- // Generate a random file name.
- fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- file, err := os.Create(fileName)
- if err != nil {
- logError(testName, function, args, startTime, "", "file create failed", err)
- return
- }
- _, err = file.Write(testCase.buf)
- if err != nil {
- logError(testName, function, args, startTime, "", "file write failed", err)
- return
- }
- file.Close()
- // Put encrypted data
- if _, err = c.FPutObject(context.Background(), bucketName, objectName, fileName, minio.PutObjectOptions{ServerSideEncryption: sse}); err != nil {
- logError(testName, function, args, startTime, "", "FPutEncryptedObject failed", err)
- return
- }
- // Read the data back
- r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err)
- return
- }
- defer r.Close()
- // Compare the sent object with the received one
- recvBuffer := bytes.NewBuffer([]byte{})
- if _, err = io.Copy(recvBuffer, r); err != nil {
- logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err)
- return
- }
- if recvBuffer.Len() != len(testCase.buf) {
- logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err)
- return
- }
- if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) {
- logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err)
- return
- }
- os.Remove(fileName)
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // testSSES3EncryptionPutGet tests SSE-S3 encryption
- func testSSES3EncryptionPutGet() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "PutEncryptedObject(bucketName, objectName, reader, sse)"
- args := map[string]interface{}{
- "bucketName": "",
- "objectName": "",
- "sse": "",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- testCases := []struct {
- buf []byte
- }{
- {buf: bytes.Repeat([]byte("F"), 1)},
- {buf: bytes.Repeat([]byte("F"), 15)},
- {buf: bytes.Repeat([]byte("F"), 16)},
- {buf: bytes.Repeat([]byte("F"), 17)},
- {buf: bytes.Repeat([]byte("F"), 31)},
- {buf: bytes.Repeat([]byte("F"), 32)},
- {buf: bytes.Repeat([]byte("F"), 33)},
- {buf: bytes.Repeat([]byte("F"), 1024)},
- {buf: bytes.Repeat([]byte("F"), 1024*2)},
- {buf: bytes.Repeat([]byte("F"), 1024*1024)},
- }
- for i, testCase := range testCases {
- // Generate a random object name
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- // Secured object
- sse := encrypt.NewSSE()
- args["sse"] = sse
- // Put encrypted data
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(testCase.buf), int64(len(testCase.buf)), minio.PutObjectOptions{ServerSideEncryption: sse})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutEncryptedObject failed", err)
- return
- }
- // Read the data back without any encryption headers
- r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err)
- return
- }
- defer r.Close()
- // Compare the sent object with the received one
- recvBuffer := bytes.NewBuffer([]byte{})
- if _, err = io.Copy(recvBuffer, r); err != nil {
- logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err)
- return
- }
- if recvBuffer.Len() != len(testCase.buf) {
- logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err)
- return
- }
- if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) {
- logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // TestSSES3EncryptionFPut tests server side encryption
- func testSSES3EncryptionFPut() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "FPutEncryptedObject(bucketName, objectName, filePath, contentType, sse)"
- args := map[string]interface{}{
- "bucketName": "",
- "objectName": "",
- "filePath": "",
- "contentType": "",
- "sse": "",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Object custom metadata
- customContentType := "custom/contenttype"
- args["metadata"] = customContentType
- testCases := []struct {
- buf []byte
- }{
- {buf: bytes.Repeat([]byte("F"), 0)},
- {buf: bytes.Repeat([]byte("F"), 1)},
- {buf: bytes.Repeat([]byte("F"), 15)},
- {buf: bytes.Repeat([]byte("F"), 16)},
- {buf: bytes.Repeat([]byte("F"), 17)},
- {buf: bytes.Repeat([]byte("F"), 31)},
- {buf: bytes.Repeat([]byte("F"), 32)},
- {buf: bytes.Repeat([]byte("F"), 33)},
- {buf: bytes.Repeat([]byte("F"), 1024)},
- {buf: bytes.Repeat([]byte("F"), 1024*2)},
- {buf: bytes.Repeat([]byte("F"), 1024*1024)},
- }
- for i, testCase := range testCases {
- // Generate a random object name
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- // Secured object
- sse := encrypt.NewSSE()
- args["sse"] = sse
- // Generate a random file name.
- fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- file, err := os.Create(fileName)
- if err != nil {
- logError(testName, function, args, startTime, "", "file create failed", err)
- return
- }
- _, err = file.Write(testCase.buf)
- if err != nil {
- logError(testName, function, args, startTime, "", "file write failed", err)
- return
- }
- file.Close()
- // Put encrypted data
- if _, err = c.FPutObject(context.Background(), bucketName, objectName, fileName, minio.PutObjectOptions{ServerSideEncryption: sse}); err != nil {
- logError(testName, function, args, startTime, "", "FPutEncryptedObject failed", err)
- return
- }
- // Read the data back
- r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err)
- return
- }
- defer r.Close()
- // Compare the sent object with the received one
- recvBuffer := bytes.NewBuffer([]byte{})
- if _, err = io.Copy(recvBuffer, r); err != nil {
- logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err)
- return
- }
- if recvBuffer.Len() != len(testCase.buf) {
- logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err)
- return
- }
- if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) {
- logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err)
- return
- }
- os.Remove(fileName)
- }
- successLogger(testName, function, args, startTime).Info()
- }
- func testBucketNotification() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "SetBucketNotification(bucketName)"
- args := map[string]interface{}{
- "bucketName": "",
- }
- if os.Getenv("NOTIFY_BUCKET") == "" ||
- os.Getenv("NOTIFY_SERVICE") == "" ||
- os.Getenv("NOTIFY_REGION") == "" ||
- os.Getenv("NOTIFY_ACCOUNTID") == "" ||
- os.Getenv("NOTIFY_RESOURCE") == "" {
- ignoredLog(testName, function, args, startTime, "Skipped notification test as it is not configured").Info()
- return
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable to debug
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- bucketName := os.Getenv("NOTIFY_BUCKET")
- args["bucketName"] = bucketName
- topicArn := notification.NewArn("aws", os.Getenv("NOTIFY_SERVICE"), os.Getenv("NOTIFY_REGION"), os.Getenv("NOTIFY_ACCOUNTID"), os.Getenv("NOTIFY_RESOURCE"))
- queueArn := notification.NewArn("aws", "dummy-service", "dummy-region", "dummy-accountid", "dummy-resource")
- topicConfig := notification.NewConfig(topicArn)
- topicConfig.AddEvents(notification.ObjectCreatedAll, notification.ObjectRemovedAll)
- topicConfig.AddFilterSuffix("jpg")
- queueConfig := notification.NewConfig(queueArn)
- queueConfig.AddEvents(notification.ObjectCreatedAll)
- queueConfig.AddFilterPrefix("photos/")
- config := notification.Configuration{}
- config.AddTopic(topicConfig)
- // Add the same topicConfig again, should have no effect
- // because it is duplicated
- config.AddTopic(topicConfig)
- if len(config.TopicConfigs) != 1 {
- logError(testName, function, args, startTime, "", "Duplicate entry added", err)
- return
- }
- // Add and remove a queue config
- config.AddQueue(queueConfig)
- config.RemoveQueueByArn(queueArn)
- err = c.SetBucketNotification(context.Background(), bucketName, config)
- if err != nil {
- logError(testName, function, args, startTime, "", "SetBucketNotification failed", err)
- return
- }
- config, err = c.GetBucketNotification(context.Background(), bucketName)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetBucketNotification failed", err)
- return
- }
- if len(config.TopicConfigs) != 1 {
- logError(testName, function, args, startTime, "", "Topic config is empty", err)
- return
- }
- if config.TopicConfigs[0].Filter.S3Key.FilterRules[0].Value != "jpg" {
- logError(testName, function, args, startTime, "", "Couldn't get the suffix", err)
- return
- }
- err = c.RemoveAllBucketNotification(context.Background(), bucketName)
- if err != nil {
- logError(testName, function, args, startTime, "", "RemoveAllBucketNotification failed", err)
- return
- }
- // Delete all objects and buckets
- if err = cleanupBucket(bucketName, c); err != nil {
- logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Tests comprehensive list of all methods.
- func testFunctional() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "testFunctional()"
- functionAll := ""
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, nil, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable to debug
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- // Make a new bucket.
- function = "MakeBucket(bucketName, region)"
- functionAll = "MakeBucket(bucketName, region)"
- args["bucketName"] = bucketName
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- defer cleanupBucket(bucketName, c)
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- // Generate a random file name.
- fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- file, err := os.Create(fileName)
- if err != nil {
- logError(testName, function, args, startTime, "", "File creation failed", err)
- return
- }
- for i := 0; i < 3; i++ {
- buf := make([]byte, rand.Intn(1<<19))
- _, err = file.Write(buf)
- if err != nil {
- logError(testName, function, args, startTime, "", "File write failed", err)
- return
- }
- }
- file.Close()
- // Verify if bucket exits and you have access.
- var exists bool
- function = "BucketExists(bucketName)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- }
- exists, err = c.BucketExists(context.Background(), bucketName)
- if err != nil {
- logError(testName, function, args, startTime, "", "BucketExists failed", err)
- return
- }
- if !exists {
- logError(testName, function, args, startTime, "", "Could not find the bucket", err)
- return
- }
- // Asserting the default bucket policy.
- function = "GetBucketPolicy(ctx, bucketName)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- }
- nilPolicy, err := c.GetBucketPolicy(context.Background(), bucketName)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err)
- return
- }
- if nilPolicy != "" {
- logError(testName, function, args, startTime, "", "policy should be set to nil", err)
- return
- }
- // Set the bucket policy to 'public readonly'.
- function = "SetBucketPolicy(bucketName, readOnlyPolicy)"
- functionAll += ", " + function
- readOnlyPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}`
- args = map[string]interface{}{
- "bucketName": bucketName,
- "bucketPolicy": readOnlyPolicy,
- }
- err = c.SetBucketPolicy(context.Background(), bucketName, readOnlyPolicy)
- if err != nil {
- logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err)
- return
- }
- // should return policy `readonly`.
- function = "GetBucketPolicy(ctx, bucketName)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- }
- _, err = c.GetBucketPolicy(context.Background(), bucketName)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err)
- return
- }
- // Make the bucket 'public writeonly'.
- function = "SetBucketPolicy(bucketName, writeOnlyPolicy)"
- functionAll += ", " + function
- writeOnlyPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}`
- args = map[string]interface{}{
- "bucketName": bucketName,
- "bucketPolicy": writeOnlyPolicy,
- }
- err = c.SetBucketPolicy(context.Background(), bucketName, writeOnlyPolicy)
- if err != nil {
- logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err)
- return
- }
- // should return policy `writeonly`.
- function = "GetBucketPolicy(ctx, bucketName)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- }
- _, err = c.GetBucketPolicy(context.Background(), bucketName)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err)
- return
- }
- // Make the bucket 'public read/write'.
- function = "SetBucketPolicy(bucketName, readWritePolicy)"
- functionAll += ", " + function
- readWritePolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}`
- args = map[string]interface{}{
- "bucketName": bucketName,
- "bucketPolicy": readWritePolicy,
- }
- err = c.SetBucketPolicy(context.Background(), bucketName, readWritePolicy)
- if err != nil {
- logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err)
- return
- }
- // should return policy `readwrite`.
- function = "GetBucketPolicy(bucketName)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- }
- _, err = c.GetBucketPolicy(context.Background(), bucketName)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err)
- return
- }
- // List all buckets.
- function = "ListBuckets()"
- functionAll += ", " + function
- args = nil
- buckets, err := c.ListBuckets(context.Background())
- if len(buckets) == 0 {
- logError(testName, function, args, startTime, "", "Found bucket list to be empty", err)
- return
- }
- if err != nil {
- logError(testName, function, args, startTime, "", "ListBuckets failed", err)
- return
- }
- // Verify if previously created bucket is listed in list buckets.
- bucketFound := false
- for _, bucket := range buckets {
- if bucket.Name == bucketName {
- bucketFound = true
- }
- }
- // If bucket not found error out.
- if !bucketFound {
- logError(testName, function, args, startTime, "", "Bucket: "+bucketName+" not found", err)
- return
- }
- objectName := bucketName + "unique"
- // Generate data
- buf := bytes.Repeat([]byte("f"), 1<<19)
- function = "PutObject(bucketName, objectName, reader, contentType)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": objectName,
- "contentType": "",
- }
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": objectName + "-nolength",
- "contentType": "binary/octet-stream",
- }
- _, err = c.PutObject(context.Background(), bucketName, objectName+"-nolength", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- // Instantiate a done channel to close all listing.
- doneCh := make(chan struct{})
- defer close(doneCh)
- objFound := false
- isRecursive := true // Recursive is true.
- function = "ListObjects(bucketName, objectName, isRecursive, doneCh)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": objectName,
- "isRecursive": isRecursive,
- }
- for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Prefix: objectName, Recursive: true}) {
- if obj.Key == objectName {
- objFound = true
- break
- }
- }
- if !objFound {
- logError(testName, function, args, startTime, "", "Object "+objectName+" not found", err)
- return
- }
- objFound = false
- isRecursive = true // Recursive is true.
- function = "ListObjects()"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": objectName,
- "isRecursive": isRecursive,
- }
- for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{Prefix: objectName, Recursive: isRecursive}) {
- if obj.Key == objectName {
- objFound = true
- break
- }
- }
- if !objFound {
- logError(testName, function, args, startTime, "", "Object "+objectName+" not found", err)
- return
- }
- incompObjNotFound := true
- function = "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": objectName,
- "isRecursive": isRecursive,
- }
- for objIncompl := range c.ListIncompleteUploads(context.Background(), bucketName, objectName, isRecursive) {
- if objIncompl.Key != "" {
- incompObjNotFound = false
- break
- }
- }
- if !incompObjNotFound {
- logError(testName, function, args, startTime, "", "Unexpected dangling incomplete upload found", err)
- return
- }
- function = "GetObject(bucketName, objectName)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": objectName,
- }
- newReader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- newReadBytes, err := ioutil.ReadAll(newReader)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAll failed", err)
- return
- }
- if !bytes.Equal(newReadBytes, buf) {
- logError(testName, function, args, startTime, "", "GetObject bytes mismatch", err)
- return
- }
- newReader.Close()
- function = "FGetObject(bucketName, objectName, fileName)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": objectName,
- "fileName": fileName + "-f",
- }
- err = c.FGetObject(context.Background(), bucketName, objectName, fileName+"-f", minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "FGetObject failed", err)
- return
- }
- function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": "",
- "expires": 3600 * time.Second,
- }
- if _, err = c.PresignedHeadObject(context.Background(), bucketName, "", 3600*time.Second, nil); err == nil {
- logError(testName, function, args, startTime, "", "PresignedHeadObject success", err)
- return
- }
- // Generate presigned HEAD object url.
- function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": objectName,
- "expires": 3600 * time.Second,
- }
- presignedHeadURL, err := c.PresignedHeadObject(context.Background(), bucketName, objectName, 3600*time.Second, nil)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedHeadObject failed", err)
- return
- }
- transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS)))
- if err != nil {
- logError(testName, function, args, startTime, "", "DefaultTransport failed", err)
- return
- }
- httpClient := &http.Client{
- // Setting a sensible time out of 30secs to wait for response
- // headers. Request is pro-actively canceled after 30secs
- // with no response.
- Timeout: 30 * time.Second,
- Transport: transport,
- }
- req, err := http.NewRequest(http.MethodHead, presignedHeadURL.String(), nil)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedHeadObject request was incorrect", err)
- return
- }
- // Verify if presigned url works.
- resp, err := httpClient.Do(req)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect", err)
- return
- }
- if resp.StatusCode != http.StatusOK {
- logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect, status "+string(resp.StatusCode), err)
- return
- }
- if resp.Header.Get("ETag") == "" {
- logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect", err)
- return
- }
- resp.Body.Close()
- function = "PresignedGetObject(bucketName, objectName, expires, reqParams)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": "",
- "expires": 3600 * time.Second,
- }
- _, err = c.PresignedGetObject(context.Background(), bucketName, "", 3600*time.Second, nil)
- if err == nil {
- logError(testName, function, args, startTime, "", "PresignedGetObject success", err)
- return
- }
- // Generate presigned GET object url.
- function = "PresignedGetObject(bucketName, objectName, expires, reqParams)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": objectName,
- "expires": 3600 * time.Second,
- }
- presignedGetURL, err := c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, nil)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedGetObject failed", err)
- return
- }
- // Verify if presigned url works.
- req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err)
- return
- }
- resp, err = httpClient.Do(req)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
- return
- }
- if resp.StatusCode != http.StatusOK {
- logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err)
- return
- }
- newPresignedBytes, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
- return
- }
- resp.Body.Close()
- if !bytes.Equal(newPresignedBytes, buf) {
- logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
- return
- }
- // Set request parameters.
- reqParams := make(url.Values)
- reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"")
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": objectName,
- "expires": 3600 * time.Second,
- "reqParams": reqParams,
- }
- presignedGetURL, err = c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, reqParams)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedGetObject failed", err)
- return
- }
- // Verify if presigned url works.
- req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err)
- return
- }
- resp, err = httpClient.Do(req)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
- return
- }
- if resp.StatusCode != http.StatusOK {
- logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err)
- return
- }
- newPresignedBytes, err = ioutil.ReadAll(resp.Body)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
- return
- }
- if !bytes.Equal(newPresignedBytes, buf) {
- logError(testName, function, args, startTime, "", "Bytes mismatch for presigned GET URL", err)
- return
- }
- if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" {
- logError(testName, function, args, startTime, "", "wrong Content-Disposition received "+string(resp.Header.Get("Content-Disposition")), err)
- return
- }
- function = "PresignedPutObject(bucketName, objectName, expires)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": "",
- "expires": 3600 * time.Second,
- }
- _, err = c.PresignedPutObject(context.Background(), bucketName, "", 3600*time.Second)
- if err == nil {
- logError(testName, function, args, startTime, "", "PresignedPutObject success", err)
- return
- }
- function = "PresignedPutObject(bucketName, objectName, expires)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": objectName + "-presigned",
- "expires": 3600 * time.Second,
- }
- presignedPutURL, err := c.PresignedPutObject(context.Background(), bucketName, objectName+"-presigned", 3600*time.Second)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedPutObject failed", err)
- return
- }
- buf = bytes.Repeat([]byte("g"), 1<<19)
- req, err = http.NewRequest(http.MethodPut, presignedPutURL.String(), bytes.NewReader(buf))
- if err != nil {
- logError(testName, function, args, startTime, "", "Couldn't make HTTP request with PresignedPutObject URL", err)
- return
- }
- resp, err = httpClient.Do(req)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedPutObject failed", err)
- return
- }
- newReader, err = c.GetObject(context.Background(), bucketName, objectName+"-presigned", minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject after PresignedPutObject failed", err)
- return
- }
- newReadBytes, err = ioutil.ReadAll(newReader)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAll after GetObject failed", err)
- return
- }
- if !bytes.Equal(newReadBytes, buf) {
- logError(testName, function, args, startTime, "", "Bytes mismatch", err)
- return
- }
- function = "RemoveObject(bucketName, objectName)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": objectName,
- }
- err = c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "RemoveObject failed", err)
- return
- }
- args["objectName"] = objectName + "-f"
- err = c.RemoveObject(context.Background(), bucketName, objectName+"-f", minio.RemoveObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "RemoveObject failed", err)
- return
- }
- args["objectName"] = objectName + "-nolength"
- err = c.RemoveObject(context.Background(), bucketName, objectName+"-nolength", minio.RemoveObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "RemoveObject failed", err)
- return
- }
- args["objectName"] = objectName + "-presigned"
- err = c.RemoveObject(context.Background(), bucketName, objectName+"-presigned", minio.RemoveObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "RemoveObject failed", err)
- return
- }
- function = "RemoveBucket(bucketName)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- }
- err = c.RemoveBucket(context.Background(), bucketName)
- if err != nil {
- logError(testName, function, args, startTime, "", "RemoveBucket failed", err)
- return
- }
- err = c.RemoveBucket(context.Background(), bucketName)
- if err == nil {
- logError(testName, function, args, startTime, "", "RemoveBucket did not fail for invalid bucket name", err)
- return
- }
- if err.Error() != "The specified bucket does not exist" {
- logError(testName, function, args, startTime, "", "RemoveBucket failed", err)
- return
- }
- os.Remove(fileName)
- os.Remove(fileName + "-f")
- successLogger(testName, functionAll, args, startTime).Info()
- }
- // Test for validating GetObject Reader* methods functioning when the
- // object is modified in the object store.
- func testGetObjectModified() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "GetObject(bucketName, objectName)"
- args := map[string]interface{}{}
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Make a new bucket.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Upload an object.
- objectName := "myobject"
- args["objectName"] = objectName
- content := "helloworld"
- _, err = c.PutObject(context.Background(), bucketName, objectName, strings.NewReader(content), int64(len(content)), minio.PutObjectOptions{ContentType: "application/text"})
- if err != nil {
- logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err)
- return
- }
- defer c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{})
- reader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "Failed to GetObject "+objectName+", from bucket "+bucketName, err)
- return
- }
- defer reader.Close()
- // Read a few bytes of the object.
- b := make([]byte, 5)
- n, err := reader.ReadAt(b, 0)
- if err != nil {
- logError(testName, function, args, startTime, "", "Failed to read object "+objectName+", from bucket "+bucketName+" at an offset", err)
- return
- }
- // Upload different contents to the same object while object is being read.
- newContent := "goodbyeworld"
- _, err = c.PutObject(context.Background(), bucketName, objectName, strings.NewReader(newContent), int64(len(newContent)), minio.PutObjectOptions{ContentType: "application/text"})
- if err != nil {
- logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err)
- return
- }
- // Confirm that a Stat() call in between doesn't change the Object's cached etag.
- _, err = reader.Stat()
- expectedError := "At least one of the pre-conditions you specified did not hold"
- if err.Error() != expectedError {
- logError(testName, function, args, startTime, "", "Expected Stat to fail with error "+expectedError+", but received "+err.Error(), err)
- return
- }
- // Read again only to find object contents have been modified since last read.
- _, err = reader.ReadAt(b, int64(n))
- if err.Error() != expectedError {
- logError(testName, function, args, startTime, "", "Expected ReadAt to fail with error "+expectedError+", but received "+err.Error(), err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test validates putObject to upload a file seeked at a given offset.
- func testPutObjectUploadSeekedObject() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "PutObject(bucketName, objectName, fileToUpload, contentType)"
- args := map[string]interface{}{
- "bucketName": "",
- "objectName": "",
- "fileToUpload": "",
- "contentType": "binary/octet-stream",
- }
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Make a new bucket.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- var tempfile *os.File
- if fileName := getMintDataDirFilePath("datafile-100-kB"); fileName != "" {
- tempfile, err = os.Open(fileName)
- if err != nil {
- logError(testName, function, args, startTime, "", "File open failed", err)
- return
- }
- args["fileToUpload"] = fileName
- } else {
- tempfile, err = ioutil.TempFile("", "minio-go-upload-test-")
- if err != nil {
- logError(testName, function, args, startTime, "", "TempFile create failed", err)
- return
- }
- args["fileToUpload"] = tempfile.Name()
- // Generate 100kB data
- if _, err = io.Copy(tempfile, getDataReader("datafile-100-kB")); err != nil {
- logError(testName, function, args, startTime, "", "File copy failed", err)
- return
- }
- defer os.Remove(tempfile.Name())
- // Seek back to the beginning of the file.
- tempfile.Seek(0, 0)
- }
- var length = 100 * humanize.KiByte
- objectName := fmt.Sprintf("test-file-%v", rand.Uint32())
- args["objectName"] = objectName
- offset := length / 2
- if _, err = tempfile.Seek(int64(offset), 0); err != nil {
- logError(testName, function, args, startTime, "", "TempFile seek failed", err)
- return
- }
- _, err = c.PutObject(context.Background(), bucketName, objectName, tempfile, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- tempfile.Close()
- obj, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- defer obj.Close()
- n, err := obj.Seek(int64(offset), 0)
- if err != nil {
- logError(testName, function, args, startTime, "", "Seek failed", err)
- return
- }
- if n != int64(offset) {
- logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(offset), n), err)
- return
- }
- _, err = c.PutObject(context.Background(), bucketName, objectName+"getobject", obj, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- st, err := c.StatObject(context.Background(), bucketName, objectName+"getobject", minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject failed", err)
- return
- }
- if st.Size != int64(length-offset) {
- logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(length-offset), n), err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Tests bucket re-create errors.
- func testMakeBucketErrorV2() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "MakeBucket(bucketName, region)"
- args := map[string]interface{}{
- "bucketName": "",
- "region": "eu-west-1",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- region := "eu-west-1"
- args["bucketName"] = bucketName
- args["region"] = region
- // Make a new bucket in 'eu-west-1'.
- if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err == nil {
- logError(testName, function, args, startTime, "", "MakeBucket did not fail for existing bucket name", err)
- return
- }
- // Verify valid error response from server.
- if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" &&
- minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" {
- logError(testName, function, args, startTime, "", "Invalid error returned by server", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test get object reader to not throw error on being closed twice.
- func testGetObjectClosedTwiceV2() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "MakeBucket(bucketName, region)"
- args := map[string]interface{}{
- "bucketName": "",
- "region": "eu-west-1",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Generate 33K of data.
- bufSize := dataFileMap["datafile-33-kB"]
- var reader = getDataReader("datafile-33-kB")
- defer reader.Close()
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- // Read the data back
- r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- st, err := r.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat failed", err)
- return
- }
- if st.Size != int64(bufSize) {
- logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(st.Size), err)
- return
- }
- if err := r.Close(); err != nil {
- logError(testName, function, args, startTime, "", "Stat failed", err)
- return
- }
- if err := r.Close(); err == nil {
- logError(testName, function, args, startTime, "", "Object is already closed, should return error", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Tests FPutObject hidden contentType setting
- func testFPutObjectV2() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "FPutObject(bucketName, objectName, fileName, opts)"
- args := map[string]interface{}{
- "bucketName": "",
- "objectName": "",
- "fileName": "",
- "opts": "",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Make a temp file with 11*1024*1024 bytes of data.
- file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest")
- if err != nil {
- logError(testName, function, args, startTime, "", "TempFile creation failed", err)
- return
- }
- r := bytes.NewReader(bytes.Repeat([]byte("b"), 11*1024*1024))
- n, err := io.CopyN(file, r, 11*1024*1024)
- if err != nil {
- logError(testName, function, args, startTime, "", "Copy failed", err)
- return
- }
- if n != int64(11*1024*1024) {
- logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err)
- return
- }
- // Close the file pro-actively for windows.
- err = file.Close()
- if err != nil {
- logError(testName, function, args, startTime, "", "File close failed", err)
- return
- }
- // Set base object name
- objectName := bucketName + "FPutObject"
- args["objectName"] = objectName
- args["fileName"] = file.Name()
- // Perform standard FPutObject with contentType provided (Expecting application/octet-stream)
- _, err = c.FPutObject(context.Background(), bucketName, objectName+"-standard", file.Name(), minio.PutObjectOptions{ContentType: "application/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "FPutObject failed", err)
- return
- }
- // Perform FPutObject with no contentType provided (Expecting application/octet-stream)
- args["objectName"] = objectName + "-Octet"
- args["contentType"] = ""
- _, err = c.FPutObject(context.Background(), bucketName, objectName+"-Octet", file.Name(), minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "FPutObject failed", err)
- return
- }
- // Add extension to temp file name
- fileName := file.Name()
- err = os.Rename(fileName, fileName+".gtar")
- if err != nil {
- logError(testName, function, args, startTime, "", "Rename failed", err)
- return
- }
- // Perform FPutObject with no contentType provided (Expecting application/x-gtar)
- args["objectName"] = objectName + "-Octet"
- args["contentType"] = ""
- args["fileName"] = fileName + ".gtar"
- _, err = c.FPutObject(context.Background(), bucketName, objectName+"-GTar", fileName+".gtar", minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "FPutObject failed", err)
- return
- }
- // Check headers and sizes
- rStandard, err := c.StatObject(context.Background(), bucketName, objectName+"-standard", minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject failed", err)
- return
- }
- if rStandard.Size != 11*1024*1024 {
- logError(testName, function, args, startTime, "", "Unexpected size", nil)
- return
- }
- if rStandard.ContentType != "application/octet-stream" {
- logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rStandard.ContentType, err)
- return
- }
- rOctet, err := c.StatObject(context.Background(), bucketName, objectName+"-Octet", minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject failed", err)
- return
- }
- if rOctet.ContentType != "application/octet-stream" {
- logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rOctet.ContentType, err)
- return
- }
- if rOctet.Size != 11*1024*1024 {
- logError(testName, function, args, startTime, "", "Unexpected size", nil)
- return
- }
- rGTar, err := c.StatObject(context.Background(), bucketName, objectName+"-GTar", minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject failed", err)
- return
- }
- if rGTar.Size != 11*1024*1024 {
- logError(testName, function, args, startTime, "", "Unexpected size", nil)
- return
- }
- if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" {
- logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/x-gtar , got "+rGTar.ContentType, err)
- return
- }
- os.Remove(fileName + ".gtar")
- successLogger(testName, function, args, startTime).Info()
- }
- // Tests various bucket supported formats.
- func testMakeBucketRegionsV2() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "MakeBucket(bucketName, region)"
- args := map[string]interface{}{
- "bucketName": "",
- "region": "eu-west-1",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket in 'eu-central-1'.
- if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "eu-west-1"}); err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- if err = cleanupBucket(bucketName, c); err != nil {
- logError(testName, function, args, startTime, "", "CleanupBucket failed while removing bucket recursively", err)
- return
- }
- // Make a new bucket with '.' in its name, in 'us-west-2'. This
- // request is internally staged into a path style instead of
- // virtual host style.
- if err = c.MakeBucket(context.Background(), bucketName+".withperiod", minio.MakeBucketOptions{Region: "us-west-2"}); err != nil {
- args["bucketName"] = bucketName + ".withperiod"
- args["region"] = "us-west-2"
- logError(testName, function, args, startTime, "", "MakeBucket test with a bucket name with period, '.', failed", err)
- return
- }
- // Delete all objects and buckets
- if err = cleanupBucket(bucketName+".withperiod", c); err != nil {
- logError(testName, function, args, startTime, "", "CleanupBucket failed while removing bucket recursively", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Tests get object ReaderSeeker interface methods.
- func testGetObjectReadSeekFunctionalV2() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "GetObject(bucketName, objectName)"
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Generate 33K of data.
- bufSize := dataFileMap["datafile-33-kB"]
- var reader = getDataReader("datafile-33-kB")
- defer reader.Close()
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- buf, err := ioutil.ReadAll(reader)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAll failed", err)
- return
- }
- // Save the data.
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- // Read the data back
- r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- defer r.Close()
- st, err := r.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat failed", err)
- return
- }
- if st.Size != int64(bufSize) {
- logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err)
- return
- }
- offset := int64(2048)
- n, err := r.Seek(offset, 0)
- if err != nil {
- logError(testName, function, args, startTime, "", "Seek failed", err)
- return
- }
- if n != offset {
- logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err)
- return
- }
- n, err = r.Seek(0, 1)
- if err != nil {
- logError(testName, function, args, startTime, "", "Seek failed", err)
- return
- }
- if n != offset {
- logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err)
- return
- }
- _, err = r.Seek(offset, 2)
- if err == nil {
- logError(testName, function, args, startTime, "", "Seek on positive offset for whence '2' should error out", err)
- return
- }
- n, err = r.Seek(-offset, 2)
- if err != nil {
- logError(testName, function, args, startTime, "", "Seek failed", err)
- return
- }
- if n != st.Size-offset {
- logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(st.Size-offset)+" got "+string(n), err)
- return
- }
- var buffer1 bytes.Buffer
- if _, err = io.CopyN(&buffer1, r, st.Size); err != nil {
- if err != io.EOF {
- logError(testName, function, args, startTime, "", "Copy failed", err)
- return
- }
- }
- if !bytes.Equal(buf[len(buf)-int(offset):], buffer1.Bytes()) {
- logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err)
- return
- }
- // Seek again and read again.
- n, err = r.Seek(offset-1, 0)
- if err != nil {
- logError(testName, function, args, startTime, "", "Seek failed", err)
- return
- }
- if n != (offset - 1) {
- logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset-1)+" got "+string(n), err)
- return
- }
- var buffer2 bytes.Buffer
- if _, err = io.CopyN(&buffer2, r, st.Size); err != nil {
- if err != io.EOF {
- logError(testName, function, args, startTime, "", "Copy failed", err)
- return
- }
- }
- // Verify now lesser bytes.
- if !bytes.Equal(buf[2047:], buffer2.Bytes()) {
- logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Tests get object ReaderAt interface methods.
- func testGetObjectReadAtFunctionalV2() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "GetObject(bucketName, objectName)"
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Generate 33K of data.
- bufSize := dataFileMap["datafile-33-kB"]
- var reader = getDataReader("datafile-33-kB")
- defer reader.Close()
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- buf, err := ioutil.ReadAll(reader)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAll failed", err)
- return
- }
- // Save the data
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- // Read the data back
- r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- defer r.Close()
- st, err := r.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat failed", err)
- return
- }
- if st.Size != int64(bufSize) {
- logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(st.Size), err)
- return
- }
- offset := int64(2048)
- // Read directly
- buf2 := make([]byte, 512)
- buf3 := make([]byte, 512)
- buf4 := make([]byte, 512)
- m, err := r.ReadAt(buf2, offset)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- if m != len(buf2) {
- logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+" got "+string(m), err)
- return
- }
- if !bytes.Equal(buf2, buf[offset:offset+512]) {
- logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
- return
- }
- offset += 512
- m, err = r.ReadAt(buf3, offset)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- if m != len(buf3) {
- logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+" got "+string(m), err)
- return
- }
- if !bytes.Equal(buf3, buf[offset:offset+512]) {
- logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
- return
- }
- offset += 512
- m, err = r.ReadAt(buf4, offset)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- if m != len(buf4) {
- logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+" got "+string(m), err)
- return
- }
- if !bytes.Equal(buf4, buf[offset:offset+512]) {
- logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err)
- return
- }
- buf5 := make([]byte, bufSize)
- // Read the whole object.
- m, err = r.ReadAt(buf5, 0)
- if err != nil {
- if err != io.EOF {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- }
- if m != len(buf5) {
- logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+" got "+string(m), err)
- return
- }
- if !bytes.Equal(buf, buf5) {
- logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err)
- return
- }
- buf6 := make([]byte, bufSize+1)
- // Read the whole object and beyond.
- _, err = r.ReadAt(buf6, 0)
- if err != nil {
- if err != io.EOF {
- logError(testName, function, args, startTime, "", "ReadAt failed", err)
- return
- }
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Tests copy object
- func testCopyObjectV2() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObject(destination, source)"
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- // Make a new bucket in 'us-east-1' (source bucket).
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Make a new bucket in 'us-east-1' (destination bucket).
- err = c.MakeBucket(context.Background(), bucketName+"-copy", minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName+"-copy", c)
- // Generate 33K of data.
- bufSize := dataFileMap["datafile-33-kB"]
- var reader = getDataReader("datafile-33-kB")
- defer reader.Close()
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- // Check the various fields of source object against destination object.
- objInfo, err := r.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat failed", err)
- return
- }
- r.Close()
- // Copy Source
- src := minio.CopySrcOptions{
- Bucket: bucketName,
- Object: objectName,
- MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC),
- MatchETag: objInfo.ETag,
- }
- args["source"] = src
- // Set copy conditions.
- dst := minio.CopyDestOptions{
- Bucket: bucketName + "-copy",
- Object: objectName + "-copy",
- }
- args["destination"] = dst
- // Perform the Copy
- _, err = c.CopyObject(context.Background(), dst, src)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObject failed", err)
- return
- }
- // Source object
- r, err = c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- // Destination object
- readerCopy, err := c.GetObject(context.Background(), bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- // Check the various fields of source object against destination object.
- objInfo, err = r.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat failed", err)
- return
- }
- objInfoCopy, err := readerCopy.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat failed", err)
- return
- }
- if objInfo.Size != objInfoCopy.Size {
- logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(objInfoCopy.Size)+" got "+string(objInfo.Size), err)
- return
- }
- // Close all the readers.
- r.Close()
- readerCopy.Close()
- // CopyObject again but with wrong conditions
- src = minio.CopySrcOptions{
- Bucket: bucketName,
- Object: objectName,
- MatchUnmodifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC),
- NoMatchETag: objInfo.ETag,
- }
- // Perform the Copy which should fail
- _, err = c.CopyObject(context.Background(), dst, src)
- if err == nil {
- logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- func testComposeObjectErrorCasesWrapper(c *minio.Client) {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "ComposeObject(destination, sourceList)"
- args := map[string]interface{}{}
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- // Make a new bucket in 'us-east-1' (source bucket).
- err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Test that more than 10K source objects cannot be
- // concatenated.
- srcArr := [10001]minio.CopySrcOptions{}
- srcSlice := srcArr[:]
- dst := minio.CopyDestOptions{
- Bucket: bucketName,
- Object: "object",
- }
- args["destination"] = dst
- // Just explain about srcArr in args["sourceList"]
- // to stop having 10,001 null headers logged
- args["sourceList"] = "source array of 10,001 elements"
- if _, err := c.ComposeObject(context.Background(), dst, srcSlice...); err == nil {
- logError(testName, function, args, startTime, "", "Expected error in ComposeObject", err)
- return
- } else if err.Error() != "There must be as least one and up to 10000 source objects." {
- logError(testName, function, args, startTime, "", "Got unexpected error", err)
- return
- }
- // Create a source with invalid offset spec and check that
- // error is returned:
- // 1. Create the source object.
- const badSrcSize = 5 * 1024 * 1024
- buf := bytes.Repeat([]byte("1"), badSrcSize)
- _, err = c.PutObject(context.Background(), bucketName, "badObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- // 2. Set invalid range spec on the object (going beyond
- // object size)
- badSrc := minio.CopySrcOptions{
- Bucket: bucketName,
- Object: "badObject",
- MatchRange: true,
- Start: 1,
- End: badSrcSize,
- }
- // 3. ComposeObject call should fail.
- if _, err := c.ComposeObject(context.Background(), dst, badSrc); err == nil {
- logError(testName, function, args, startTime, "", "ComposeObject expected to fail", err)
- return
- } else if !strings.Contains(err.Error(), "has invalid segment-to-copy") {
- logError(testName, function, args, startTime, "", "Got invalid error", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test expected error cases
- func testComposeObjectErrorCasesV2() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "ComposeObject(destination, sourceList)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
- return
- }
- testComposeObjectErrorCasesWrapper(c)
- }
- func testComposeMultipleSources(c *minio.Client) {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "ComposeObject(destination, sourceList)"
- args := map[string]interface{}{
- "destination": "",
- "sourceList": "",
- }
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- // Make a new bucket in 'us-east-1' (source bucket).
- err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Upload a small source object
- const srcSize = 1024 * 1024 * 5
- buf := bytes.Repeat([]byte("1"), srcSize)
- _, err = c.PutObject(context.Background(), bucketName, "srcObject", bytes.NewReader(buf), int64(srcSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- // We will append 10 copies of the object.
- srcs := []minio.CopySrcOptions{}
- for i := 0; i < 10; i++ {
- srcs = append(srcs, minio.CopySrcOptions{
- Bucket: bucketName,
- Object: "srcObject",
- })
- }
- // make the last part very small
- srcs[9].MatchRange = true
- args["sourceList"] = srcs
- dst := minio.CopyDestOptions{
- Bucket: bucketName,
- Object: "dstObject",
- }
- args["destination"] = dst
- ui, err := c.ComposeObject(context.Background(), dst, srcs...)
- if err != nil {
- logError(testName, function, args, startTime, "", "ComposeObject failed", err)
- return
- }
- if ui.Size != 9*srcSize+1 {
- logError(testName, function, args, startTime, "", "ComposeObject returned unexpected size", err)
- return
- }
- objProps, err := c.StatObject(context.Background(), bucketName, "dstObject", minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject failed", err)
- return
- }
- if objProps.Size != 9*srcSize+1 {
- logError(testName, function, args, startTime, "", "Size mismatched! Expected "+string(10000*srcSize)+" got "+string(objProps.Size), err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test concatenating multiple 10K objects V2
- func testCompose10KSourcesV2() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "ComposeObject(destination, sourceList)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
- return
- }
- testComposeMultipleSources(c)
- }
- func testEncryptedEmptyObject() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "PutObject(bucketName, objectName, reader, objectSize, opts)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
- return
- }
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket in 'us-east-1' (source bucket).
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- sse := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"object"))
- // 1. create an sse-c encrypted object to copy by uploading
- const srcSize = 0
- var buf []byte // Empty buffer
- args["objectName"] = "object"
- _, err = c.PutObject(context.Background(), bucketName, "object", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ServerSideEncryption: sse})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject call failed", err)
- return
- }
- // 2. Test CopyObject for an empty object
- src := minio.CopySrcOptions{
- Bucket: bucketName,
- Object: "object",
- Encryption: sse,
- }
- dst := minio.CopyDestOptions{
- Bucket: bucketName,
- Object: "new-object",
- Encryption: sse,
- }
- if _, err = c.CopyObject(context.Background(), dst, src); err != nil {
- function = "CopyObject(dst, src)"
- logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject failed", err)
- return
- }
- // 3. Test Key rotation
- newSSE := encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"new-object"))
- src = minio.CopySrcOptions{
- Bucket: bucketName,
- Object: "new-object",
- Encryption: sse,
- }
- dst = minio.CopyDestOptions{
- Bucket: bucketName,
- Object: "new-object",
- Encryption: newSSE,
- }
- if _, err = c.CopyObject(context.Background(), dst, src); err != nil {
- function = "CopyObject(dst, src)"
- logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject with key rotation failed", err)
- return
- }
- // 4. Download the object.
- reader, err := c.GetObject(context.Background(), bucketName, "new-object", minio.GetObjectOptions{ServerSideEncryption: newSSE})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- defer reader.Close()
- decBytes, err := ioutil.ReadAll(reader)
- if err != nil {
- logError(testName, function, map[string]interface{}{}, startTime, "", "ReadAll failed", err)
- return
- }
- if !bytes.Equal(decBytes, buf) {
- logError(testName, function, map[string]interface{}{}, startTime, "", "Downloaded object doesn't match the empty encrypted object", err)
- return
- }
- delete(args, "objectName")
- successLogger(testName, function, args, startTime).Info()
- }
- func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, sseDst encrypt.ServerSide) {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncNameLoc(2)
- function := "CopyObject(destination, source)"
- args := map[string]interface{}{}
- var srcEncryption, dstEncryption encrypt.ServerSide
- // Make a new bucket in 'us-east-1' (source bucket).
- err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // 1. create an sse-c encrypted object to copy by uploading
- const srcSize = 1024 * 1024
- buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB
- _, err = c.PutObject(context.Background(), bucketName, "srcObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{
- ServerSideEncryption: sseSrc,
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject call failed", err)
- return
- }
- if sseSrc != nil && sseSrc.Type() != encrypt.S3 {
- srcEncryption = sseSrc
- }
- // 2. copy object and change encryption key
- src := minio.CopySrcOptions{
- Bucket: bucketName,
- Object: "srcObject",
- Encryption: srcEncryption,
- }
- args["source"] = src
- dst := minio.CopyDestOptions{
- Bucket: bucketName,
- Object: "dstObject",
- Encryption: sseDst,
- }
- args["destination"] = dst
- _, err = c.CopyObject(context.Background(), dst, src)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObject failed", err)
- return
- }
- if sseDst != nil && sseDst.Type() != encrypt.S3 {
- dstEncryption = sseDst
- }
- // 3. get copied object and check if content is equal
- coreClient := minio.Core{c}
- reader, _, _, err := coreClient.GetObject(context.Background(), bucketName, "dstObject", minio.GetObjectOptions{ServerSideEncryption: dstEncryption})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- decBytes, err := ioutil.ReadAll(reader)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAll failed", err)
- return
- }
- if !bytes.Equal(decBytes, buf) {
- logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err)
- return
- }
- reader.Close()
- // Test key rotation for source object in-place.
- var newSSE encrypt.ServerSide
- if sseSrc != nil && sseSrc.Type() == encrypt.SSEC {
- newSSE = encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"srcObject")) // replace key
- }
- if sseSrc != nil && sseSrc.Type() == encrypt.S3 {
- newSSE = encrypt.NewSSE()
- }
- if newSSE != nil {
- dst = minio.CopyDestOptions{
- Bucket: bucketName,
- Object: "srcObject",
- Encryption: newSSE,
- }
- args["destination"] = dst
- _, err = c.CopyObject(context.Background(), dst, src)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObject failed", err)
- return
- }
- // Get copied object and check if content is equal
- reader, _, _, err = coreClient.GetObject(context.Background(), bucketName, "srcObject", minio.GetObjectOptions{ServerSideEncryption: newSSE})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- decBytes, err = ioutil.ReadAll(reader)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAll failed", err)
- return
- }
- if !bytes.Equal(decBytes, buf) {
- logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err)
- return
- }
- reader.Close()
- // Test in-place decryption.
- dst = minio.CopyDestOptions{
- Bucket: bucketName,
- Object: "srcObject",
- }
- args["destination"] = dst
- src = minio.CopySrcOptions{
- Bucket: bucketName,
- Object: "srcObject",
- Encryption: newSSE,
- }
- args["source"] = src
- _, err = c.CopyObject(context.Background(), dst, src)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObject Key rotation failed", err)
- return
- }
- }
- // Get copied decrypted object and check if content is equal
- reader, _, _, err = coreClient.GetObject(context.Background(), bucketName, "srcObject", minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- defer reader.Close()
- decBytes, err = ioutil.ReadAll(reader)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAll failed", err)
- return
- }
- if !bytes.Equal(decBytes, buf) {
- logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test encrypted copy object
- func testUnencryptedToSSECCopyObject() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObject(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
- return
- }
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject"))
- // c.TraceOn(os.Stderr)
- testEncryptedCopyObjectWrapper(c, bucketName, nil, sseDst)
- }
- // Test encrypted copy object
- func testUnencryptedToSSES3CopyObject() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObject(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
- return
- }
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- var sseSrc encrypt.ServerSide
- sseDst := encrypt.NewSSE()
- // c.TraceOn(os.Stderr)
- testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
- }
- // Test encrypted copy object
- func testUnencryptedToUnencryptedCopyObject() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObject(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
- return
- }
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- var sseSrc, sseDst encrypt.ServerSide
- // c.TraceOn(os.Stderr)
- testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
- }
- // Test encrypted copy object
- func testEncryptedSSECToSSECCopyObject() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObject(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
- return
- }
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject"))
- sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject"))
- // c.TraceOn(os.Stderr)
- testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
- }
- // Test encrypted copy object
- func testEncryptedSSECToSSES3CopyObject() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObject(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
- return
- }
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject"))
- sseDst := encrypt.NewSSE()
- // c.TraceOn(os.Stderr)
- testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
- }
- // Test encrypted copy object
- func testEncryptedSSECToUnencryptedCopyObject() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObject(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
- return
- }
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject"))
- var sseDst encrypt.ServerSide
- // c.TraceOn(os.Stderr)
- testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
- }
- // Test encrypted copy object
- func testEncryptedSSES3ToSSECCopyObject() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObject(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
- return
- }
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- sseSrc := encrypt.NewSSE()
- sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject"))
- // c.TraceOn(os.Stderr)
- testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
- }
- // Test encrypted copy object
- func testEncryptedSSES3ToSSES3CopyObject() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObject(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
- return
- }
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- sseSrc := encrypt.NewSSE()
- sseDst := encrypt.NewSSE()
- // c.TraceOn(os.Stderr)
- testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
- }
- // Test encrypted copy object
- func testEncryptedSSES3ToUnencryptedCopyObject() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObject(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
- return
- }
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- sseSrc := encrypt.NewSSE()
- var sseDst encrypt.ServerSide
- // c.TraceOn(os.Stderr)
- testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
- }
- // Test encrypted copy object
- func testEncryptedCopyObjectV2() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObject(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
- return
- }
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject"))
- sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject"))
- // c.TraceOn(os.Stderr)
- testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst)
- }
- func testDecryptedCopyObject() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObject(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
- return
- }
- bucketName, objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-"), "object"
- if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}); err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- encryption := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName))
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(bytes.Repeat([]byte("a"), 1024*1024)), 1024*1024, minio.PutObjectOptions{
- ServerSideEncryption: encryption,
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject call failed", err)
- return
- }
- src := minio.CopySrcOptions{
- Bucket: bucketName,
- Object: objectName,
- Encryption: encrypt.SSECopy(encryption),
- }
- args["source"] = src
- dst := minio.CopyDestOptions{
- Bucket: bucketName,
- Object: "decrypted-" + objectName,
- }
- args["destination"] = dst
- if _, err = c.CopyObject(context.Background(), dst, src); err != nil {
- logError(testName, function, args, startTime, "", "CopyObject failed", err)
- return
- }
- if _, err = c.GetObject(context.Background(), bucketName, "decrypted-"+objectName, minio.GetObjectOptions{}); err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- func testSSECMultipartEncryptedToSSECCopyObjectPart() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObjectPart(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- client, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
- return
- }
- // Instantiate new core client object.
- c := minio.Core{client}
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, client)
- // Make a buffer with 6MB of data
- buf := bytes.Repeat([]byte("abcdef"), 1024*1024)
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- password := "correct horse battery staple"
- srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName))
- // Upload a 6MB object using multipart mechanism
- uploadID, err := c.NewMultipartUpload(context.Background(), bucketName, objectName, minio.PutObjectOptions{ServerSideEncryption: srcencryption})
- if err != nil {
- logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
- return
- }
- var completeParts []minio.CompletePart
- part, err := c.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, bytes.NewReader(buf[:5*1024*1024]), 5*1024*1024, "", "", srcencryption)
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObjectPart call failed", err)
- return
- }
- completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag})
- part, err = c.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 2, bytes.NewReader(buf[5*1024*1024:]), 1024*1024, "", "", srcencryption)
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObjectPart call failed", err)
- return
- }
- completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag})
- // Complete the multipart upload
- _, err = c.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, completeParts, minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
- return
- }
- // Stat the object and check its length matches
- objInfo, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject call failed", err)
- return
- }
- destBucketName := bucketName
- destObjectName := objectName + "-dest"
- dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName))
- uploadID, err = c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
- if err != nil {
- logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
- return
- }
- // Content of the destination object will be two copies of
- // `objectName` concatenated, followed by first byte of
- // `objectName`.
- metadata := make(map[string]string)
- header := make(http.Header)
- encrypt.SSECopy(srcencryption).Marshal(header)
- dstencryption.Marshal(header)
- for k, v := range header {
- metadata[k] = v[0]
- }
- metadata["x-amz-copy-source-if-match"] = objInfo.ETag
- // First of three parts
- fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Second of three parts
- sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Last of three parts
- lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Complete the multipart upload
- _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
- return
- }
- // Stat the object and check its length matches
- objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject call failed", err)
- return
- }
- if objInfo.Size != (6*1024*1024)*2+1 {
- logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
- return
- }
- // Now we read the data back
- getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption}
- getOpts.SetRange(0, 6*1024*1024-1)
- r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject call failed", err)
- return
- }
- getBuf := make([]byte, 6*1024*1024)
- _, err = readFull(r, getBuf)
- if err != nil {
- logError(testName, function, args, startTime, "", "Read buffer failed", err)
- return
- }
- if !bytes.Equal(getBuf, buf) {
- logError(testName, function, args, startTime, "", "Got unexpected data in first 6MB", err)
- return
- }
- getOpts.SetRange(6*1024*1024, 0)
- r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject call failed", err)
- return
- }
- getBuf = make([]byte, 6*1024*1024+1)
- _, err = readFull(r, getBuf)
- if err != nil {
- logError(testName, function, args, startTime, "", "Read buffer failed", err)
- return
- }
- if !bytes.Equal(getBuf[:6*1024*1024], buf) {
- logError(testName, function, args, startTime, "", "Got unexpected data in second 6MB", err)
- return
- }
- if getBuf[6*1024*1024] != buf[0] {
- logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- // Do not need to remove destBucketName its same as bucketName.
- }
- // Test Core CopyObjectPart implementation
- func testSSECEncryptedToSSECCopyObjectPart() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObjectPart(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- client, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
- return
- }
- // Instantiate new core client object.
- c := minio.Core{client}
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, client)
- // Make a buffer with 5MB of data
- buf := bytes.Repeat([]byte("abcde"), 1024*1024)
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- password := "correct horse battery staple"
- srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName))
- putmetadata := map[string]string{
- "Content-Type": "binary/octet-stream",
- }
- opts := minio.PutObjectOptions{
- UserMetadata: putmetadata,
- ServerSideEncryption: srcencryption,
- }
- uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject call failed", err)
- return
- }
- st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject call failed", err)
- return
- }
- if st.Size != int64(len(buf)) {
- logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
- return
- }
- destBucketName := bucketName
- destObjectName := objectName + "-dest"
- dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName))
- uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
- if err != nil {
- logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
- return
- }
- // Content of the destination object will be two copies of
- // `objectName` concatenated, followed by first byte of
- // `objectName`.
- metadata := make(map[string]string)
- header := make(http.Header)
- encrypt.SSECopy(srcencryption).Marshal(header)
- dstencryption.Marshal(header)
- for k, v := range header {
- metadata[k] = v[0]
- }
- metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
- // First of three parts
- fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Second of three parts
- sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Last of three parts
- lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Complete the multipart upload
- _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
- return
- }
- // Stat the object and check its length matches
- objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject call failed", err)
- return
- }
- if objInfo.Size != (5*1024*1024)*2+1 {
- logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
- return
- }
- // Now we read the data back
- getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption}
- getOpts.SetRange(0, 5*1024*1024-1)
- r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject call failed", err)
- return
- }
- getBuf := make([]byte, 5*1024*1024)
- _, err = readFull(r, getBuf)
- if err != nil {
- logError(testName, function, args, startTime, "", "Read buffer failed", err)
- return
- }
- if !bytes.Equal(getBuf, buf) {
- logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
- return
- }
- getOpts.SetRange(5*1024*1024, 0)
- r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject call failed", err)
- return
- }
- getBuf = make([]byte, 5*1024*1024+1)
- _, err = readFull(r, getBuf)
- if err != nil {
- logError(testName, function, args, startTime, "", "Read buffer failed", err)
- return
- }
- if !bytes.Equal(getBuf[:5*1024*1024], buf) {
- logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
- return
- }
- if getBuf[5*1024*1024] != buf[0] {
- logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- // Do not need to remove destBucketName its same as bucketName.
- }
- // Test Core CopyObjectPart implementation for SSEC encrypted to unencrypted copy
- func testSSECEncryptedToUnencryptedCopyPart() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObjectPart(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- client, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
- return
- }
- // Instantiate new core client object.
- c := minio.Core{client}
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, client)
- // Make a buffer with 5MB of data
- buf := bytes.Repeat([]byte("abcde"), 1024*1024)
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- password := "correct horse battery staple"
- srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName))
- opts := minio.PutObjectOptions{
- UserMetadata: map[string]string{
- "Content-Type": "binary/octet-stream",
- },
- ServerSideEncryption: srcencryption,
- }
- uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject call failed", err)
- return
- }
- st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject call failed", err)
- return
- }
- if st.Size != int64(len(buf)) {
- logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
- return
- }
- destBucketName := bucketName
- destObjectName := objectName + "-dest"
- var dstencryption encrypt.ServerSide
- uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
- if err != nil {
- logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
- return
- }
- // Content of the destination object will be two copies of
- // `objectName` concatenated, followed by first byte of
- // `objectName`.
- metadata := make(map[string]string)
- header := make(http.Header)
- encrypt.SSECopy(srcencryption).Marshal(header)
- for k, v := range header {
- metadata[k] = v[0]
- }
- metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
- // First of three parts
- fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Second of three parts
- sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Last of three parts
- lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Complete the multipart upload
- _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
- return
- }
- // Stat the object and check its length matches
- objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject call failed", err)
- return
- }
- if objInfo.Size != (5*1024*1024)*2+1 {
- logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
- return
- }
- // Now we read the data back
- getOpts := minio.GetObjectOptions{}
- getOpts.SetRange(0, 5*1024*1024-1)
- r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject call failed", err)
- return
- }
- getBuf := make([]byte, 5*1024*1024)
- _, err = readFull(r, getBuf)
- if err != nil {
- logError(testName, function, args, startTime, "", "Read buffer failed", err)
- return
- }
- if !bytes.Equal(getBuf, buf) {
- logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
- return
- }
- getOpts.SetRange(5*1024*1024, 0)
- r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject call failed", err)
- return
- }
- getBuf = make([]byte, 5*1024*1024+1)
- _, err = readFull(r, getBuf)
- if err != nil {
- logError(testName, function, args, startTime, "", "Read buffer failed", err)
- return
- }
- if !bytes.Equal(getBuf[:5*1024*1024], buf) {
- logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
- return
- }
- if getBuf[5*1024*1024] != buf[0] {
- logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- // Do not need to remove destBucketName its same as bucketName.
- }
- // Test Core CopyObjectPart implementation for SSEC encrypted to SSE-S3 encrypted copy
- func testSSECEncryptedToSSES3CopyObjectPart() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObjectPart(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- client, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
- return
- }
- // Instantiate new core client object.
- c := minio.Core{client}
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, client)
- // Make a buffer with 5MB of data
- buf := bytes.Repeat([]byte("abcde"), 1024*1024)
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- password := "correct horse battery staple"
- srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName))
- putmetadata := map[string]string{
- "Content-Type": "binary/octet-stream",
- }
- opts := minio.PutObjectOptions{
- UserMetadata: putmetadata,
- ServerSideEncryption: srcencryption,
- }
- uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject call failed", err)
- return
- }
- st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject call failed", err)
- return
- }
- if st.Size != int64(len(buf)) {
- logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
- return
- }
- destBucketName := bucketName
- destObjectName := objectName + "-dest"
- dstencryption := encrypt.NewSSE()
- uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
- if err != nil {
- logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
- return
- }
- // Content of the destination object will be two copies of
- // `objectName` concatenated, followed by first byte of
- // `objectName`.
- metadata := make(map[string]string)
- header := make(http.Header)
- encrypt.SSECopy(srcencryption).Marshal(header)
- dstencryption.Marshal(header)
- for k, v := range header {
- metadata[k] = v[0]
- }
- metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
- // First of three parts
- fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Second of three parts
- sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Last of three parts
- lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Complete the multipart upload
- _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
- return
- }
- // Stat the object and check its length matches
- objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject call failed", err)
- return
- }
- if objInfo.Size != (5*1024*1024)*2+1 {
- logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
- return
- }
- // Now we read the data back
- getOpts := minio.GetObjectOptions{}
- getOpts.SetRange(0, 5*1024*1024-1)
- r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject call failed", err)
- return
- }
- getBuf := make([]byte, 5*1024*1024)
- _, err = readFull(r, getBuf)
- if err != nil {
- logError(testName, function, args, startTime, "", "Read buffer failed", err)
- return
- }
- if !bytes.Equal(getBuf, buf) {
- logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
- return
- }
- getOpts.SetRange(5*1024*1024, 0)
- r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject call failed", err)
- return
- }
- getBuf = make([]byte, 5*1024*1024+1)
- _, err = readFull(r, getBuf)
- if err != nil {
- logError(testName, function, args, startTime, "", "Read buffer failed", err)
- return
- }
- if !bytes.Equal(getBuf[:5*1024*1024], buf) {
- logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
- return
- }
- if getBuf[5*1024*1024] != buf[0] {
- logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- // Do not need to remove destBucketName its same as bucketName.
- }
- // Test Core CopyObjectPart implementation for unencrypted to SSEC encryption copy part
- func testUnencryptedToSSECCopyObjectPart() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObjectPart(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- client, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
- return
- }
- // Instantiate new core client object.
- c := minio.Core{client}
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, client)
- // Make a buffer with 5MB of data
- buf := bytes.Repeat([]byte("abcde"), 1024*1024)
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- password := "correct horse battery staple"
- putmetadata := map[string]string{
- "Content-Type": "binary/octet-stream",
- }
- opts := minio.PutObjectOptions{
- UserMetadata: putmetadata,
- }
- uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject call failed", err)
- return
- }
- st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject call failed", err)
- return
- }
- if st.Size != int64(len(buf)) {
- logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
- return
- }
- destBucketName := bucketName
- destObjectName := objectName + "-dest"
- dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName))
- uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
- if err != nil {
- logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
- return
- }
- // Content of the destination object will be two copies of
- // `objectName` concatenated, followed by first byte of
- // `objectName`.
- metadata := make(map[string]string)
- header := make(http.Header)
- dstencryption.Marshal(header)
- for k, v := range header {
- metadata[k] = v[0]
- }
- metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
- // First of three parts
- fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Second of three parts
- sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Last of three parts
- lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Complete the multipart upload
- _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
- return
- }
- // Stat the object and check its length matches
- objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject call failed", err)
- return
- }
- if objInfo.Size != (5*1024*1024)*2+1 {
- logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
- return
- }
- // Now we read the data back
- getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption}
- getOpts.SetRange(0, 5*1024*1024-1)
- r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject call failed", err)
- return
- }
- getBuf := make([]byte, 5*1024*1024)
- _, err = readFull(r, getBuf)
- if err != nil {
- logError(testName, function, args, startTime, "", "Read buffer failed", err)
- return
- }
- if !bytes.Equal(getBuf, buf) {
- logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
- return
- }
- getOpts.SetRange(5*1024*1024, 0)
- r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject call failed", err)
- return
- }
- getBuf = make([]byte, 5*1024*1024+1)
- _, err = readFull(r, getBuf)
- if err != nil {
- logError(testName, function, args, startTime, "", "Read buffer failed", err)
- return
- }
- if !bytes.Equal(getBuf[:5*1024*1024], buf) {
- logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
- return
- }
- if getBuf[5*1024*1024] != buf[0] {
- logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- // Do not need to remove destBucketName its same as bucketName.
- }
- // Test Core CopyObjectPart implementation for unencrypted to unencrypted copy
- func testUnencryptedToUnencryptedCopyPart() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObjectPart(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- client, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
- return
- }
- // Instantiate new core client object.
- c := minio.Core{client}
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, client)
- // Make a buffer with 5MB of data
- buf := bytes.Repeat([]byte("abcde"), 1024*1024)
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- putmetadata := map[string]string{
- "Content-Type": "binary/octet-stream",
- }
- opts := minio.PutObjectOptions{
- UserMetadata: putmetadata,
- }
- uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject call failed", err)
- return
- }
- st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject call failed", err)
- return
- }
- if st.Size != int64(len(buf)) {
- logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
- return
- }
- destBucketName := bucketName
- destObjectName := objectName + "-dest"
- uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
- return
- }
- // Content of the destination object will be two copies of
- // `objectName` concatenated, followed by first byte of
- // `objectName`.
- metadata := make(map[string]string)
- header := make(http.Header)
- for k, v := range header {
- metadata[k] = v[0]
- }
- metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
- // First of three parts
- fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Second of three parts
- sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Last of three parts
- lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Complete the multipart upload
- _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
- return
- }
- // Stat the object and check its length matches
- objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject call failed", err)
- return
- }
- if objInfo.Size != (5*1024*1024)*2+1 {
- logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
- return
- }
- // Now we read the data back
- getOpts := minio.GetObjectOptions{}
- getOpts.SetRange(0, 5*1024*1024-1)
- r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject call failed", err)
- return
- }
- getBuf := make([]byte, 5*1024*1024)
- _, err = readFull(r, getBuf)
- if err != nil {
- logError(testName, function, args, startTime, "", "Read buffer failed", err)
- return
- }
- if !bytes.Equal(getBuf, buf) {
- logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
- return
- }
- getOpts.SetRange(5*1024*1024, 0)
- r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject call failed", err)
- return
- }
- getBuf = make([]byte, 5*1024*1024+1)
- _, err = readFull(r, getBuf)
- if err != nil {
- logError(testName, function, args, startTime, "", "Read buffer failed", err)
- return
- }
- if !bytes.Equal(getBuf[:5*1024*1024], buf) {
- logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
- return
- }
- if getBuf[5*1024*1024] != buf[0] {
- logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- // Do not need to remove destBucketName its same as bucketName.
- }
- // Test Core CopyObjectPart implementation for unencrypted to SSE-S3 encrypted copy
- func testUnencryptedToSSES3CopyObjectPart() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObjectPart(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- client, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
- return
- }
- // Instantiate new core client object.
- c := minio.Core{client}
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, client)
- // Make a buffer with 5MB of data
- buf := bytes.Repeat([]byte("abcde"), 1024*1024)
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- opts := minio.PutObjectOptions{
- UserMetadata: map[string]string{
- "Content-Type": "binary/octet-stream",
- },
- }
- uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject call failed", err)
- return
- }
- st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject call failed", err)
- return
- }
- if st.Size != int64(len(buf)) {
- logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
- return
- }
- destBucketName := bucketName
- destObjectName := objectName + "-dest"
- dstencryption := encrypt.NewSSE()
- uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
- if err != nil {
- logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
- return
- }
- // Content of the destination object will be two copies of
- // `objectName` concatenated, followed by first byte of
- // `objectName`.
- metadata := make(map[string]string)
- header := make(http.Header)
- dstencryption.Marshal(header)
- for k, v := range header {
- metadata[k] = v[0]
- }
- metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
- // First of three parts
- fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Second of three parts
- sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Last of three parts
- lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Complete the multipart upload
- _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
- return
- }
- // Stat the object and check its length matches
- objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject call failed", err)
- return
- }
- if objInfo.Size != (5*1024*1024)*2+1 {
- logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
- return
- }
- // Now we read the data back
- getOpts := minio.GetObjectOptions{}
- getOpts.SetRange(0, 5*1024*1024-1)
- r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject call failed", err)
- return
- }
- getBuf := make([]byte, 5*1024*1024)
- _, err = readFull(r, getBuf)
- if err != nil {
- logError(testName, function, args, startTime, "", "Read buffer failed", err)
- return
- }
- if !bytes.Equal(getBuf, buf) {
- logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
- return
- }
- getOpts.SetRange(5*1024*1024, 0)
- r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject call failed", err)
- return
- }
- getBuf = make([]byte, 5*1024*1024+1)
- _, err = readFull(r, getBuf)
- if err != nil {
- logError(testName, function, args, startTime, "", "Read buffer failed", err)
- return
- }
- if !bytes.Equal(getBuf[:5*1024*1024], buf) {
- logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
- return
- }
- if getBuf[5*1024*1024] != buf[0] {
- logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- // Do not need to remove destBucketName its same as bucketName.
- }
- // Test Core CopyObjectPart implementation for SSE-S3 to SSEC encryption copy part
- func testSSES3EncryptedToSSECCopyObjectPart() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObjectPart(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- client, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
- return
- }
- // Instantiate new core client object.
- c := minio.Core{client}
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, client)
- // Make a buffer with 5MB of data
- buf := bytes.Repeat([]byte("abcde"), 1024*1024)
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- password := "correct horse battery staple"
- srcEncryption := encrypt.NewSSE()
- opts := minio.PutObjectOptions{
- UserMetadata: map[string]string{
- "Content-Type": "binary/octet-stream",
- },
- ServerSideEncryption: srcEncryption,
- }
- uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject call failed", err)
- return
- }
- st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcEncryption})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject call failed", err)
- return
- }
- if st.Size != int64(len(buf)) {
- logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
- return
- }
- destBucketName := bucketName
- destObjectName := objectName + "-dest"
- dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName))
- uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
- if err != nil {
- logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
- return
- }
- // Content of the destination object will be two copies of
- // `objectName` concatenated, followed by first byte of
- // `objectName`.
- metadata := make(map[string]string)
- header := make(http.Header)
- dstencryption.Marshal(header)
- for k, v := range header {
- metadata[k] = v[0]
- }
- metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
- // First of three parts
- fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Second of three parts
- sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Last of three parts
- lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Complete the multipart upload
- _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
- return
- }
- // Stat the object and check its length matches
- objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject call failed", err)
- return
- }
- if objInfo.Size != (5*1024*1024)*2+1 {
- logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
- return
- }
- // Now we read the data back
- getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption}
- getOpts.SetRange(0, 5*1024*1024-1)
- r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject call failed", err)
- return
- }
- getBuf := make([]byte, 5*1024*1024)
- _, err = readFull(r, getBuf)
- if err != nil {
- logError(testName, function, args, startTime, "", "Read buffer failed", err)
- return
- }
- if !bytes.Equal(getBuf, buf) {
- logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
- return
- }
- getOpts.SetRange(5*1024*1024, 0)
- r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject call failed", err)
- return
- }
- getBuf = make([]byte, 5*1024*1024+1)
- _, err = readFull(r, getBuf)
- if err != nil {
- logError(testName, function, args, startTime, "", "Read buffer failed", err)
- return
- }
- if !bytes.Equal(getBuf[:5*1024*1024], buf) {
- logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
- return
- }
- if getBuf[5*1024*1024] != buf[0] {
- logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- // Do not need to remove destBucketName its same as bucketName.
- }
- // Test Core CopyObjectPart implementation for unencrypted to unencrypted copy
- func testSSES3EncryptedToUnencryptedCopyPart() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObjectPart(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- client, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
- return
- }
- // Instantiate new core client object.
- c := minio.Core{client}
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, client)
- // Make a buffer with 5MB of data
- buf := bytes.Repeat([]byte("abcde"), 1024*1024)
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- srcEncryption := encrypt.NewSSE()
- opts := minio.PutObjectOptions{
- UserMetadata: map[string]string{
- "Content-Type": "binary/octet-stream",
- },
- ServerSideEncryption: srcEncryption,
- }
- uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject call failed", err)
- return
- }
- st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcEncryption})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject call failed", err)
- return
- }
- if st.Size != int64(len(buf)) {
- logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
- return
- }
- destBucketName := bucketName
- destObjectName := objectName + "-dest"
- uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
- return
- }
- // Content of the destination object will be two copies of
- // `objectName` concatenated, followed by first byte of
- // `objectName`.
- metadata := make(map[string]string)
- header := make(http.Header)
- for k, v := range header {
- metadata[k] = v[0]
- }
- metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
- // First of three parts
- fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Second of three parts
- sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Last of three parts
- lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Complete the multipart upload
- _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
- return
- }
- // Stat the object and check its length matches
- objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject call failed", err)
- return
- }
- if objInfo.Size != (5*1024*1024)*2+1 {
- logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
- return
- }
- // Now we read the data back
- getOpts := minio.GetObjectOptions{}
- getOpts.SetRange(0, 5*1024*1024-1)
- r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject call failed", err)
- return
- }
- getBuf := make([]byte, 5*1024*1024)
- _, err = readFull(r, getBuf)
- if err != nil {
- logError(testName, function, args, startTime, "", "Read buffer failed", err)
- return
- }
- if !bytes.Equal(getBuf, buf) {
- logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
- return
- }
- getOpts.SetRange(5*1024*1024, 0)
- r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject call failed", err)
- return
- }
- getBuf = make([]byte, 5*1024*1024+1)
- _, err = readFull(r, getBuf)
- if err != nil {
- logError(testName, function, args, startTime, "", "Read buffer failed", err)
- return
- }
- if !bytes.Equal(getBuf[:5*1024*1024], buf) {
- logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
- return
- }
- if getBuf[5*1024*1024] != buf[0] {
- logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- // Do not need to remove destBucketName its same as bucketName.
- }
- // Test Core CopyObjectPart implementation for unencrypted to SSE-S3 encrypted copy
- func testSSES3EncryptedToSSES3CopyObjectPart() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObjectPart(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- client, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
- return
- }
- // Instantiate new core client object.
- c := minio.Core{client}
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, client)
- // Make a buffer with 5MB of data
- buf := bytes.Repeat([]byte("abcde"), 1024*1024)
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- srcEncryption := encrypt.NewSSE()
- opts := minio.PutObjectOptions{
- UserMetadata: map[string]string{
- "Content-Type": "binary/octet-stream",
- },
- ServerSideEncryption: srcEncryption,
- }
- uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts)
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject call failed", err)
- return
- }
- st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcEncryption})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject call failed", err)
- return
- }
- if st.Size != int64(len(buf)) {
- logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err)
- return
- }
- destBucketName := bucketName
- destObjectName := objectName + "-dest"
- dstencryption := encrypt.NewSSE()
- uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption})
- if err != nil {
- logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err)
- return
- }
- // Content of the destination object will be two copies of
- // `objectName` concatenated, followed by first byte of
- // `objectName`.
- metadata := make(map[string]string)
- header := make(http.Header)
- dstencryption.Marshal(header)
- for k, v := range header {
- metadata[k] = v[0]
- }
- metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag
- // First of three parts
- fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Second of three parts
- sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Last of three parts
- lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err)
- return
- }
- // Complete the multipart upload
- _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err)
- return
- }
- // Stat the object and check its length matches
- objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject call failed", err)
- return
- }
- if objInfo.Size != (5*1024*1024)*2+1 {
- logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err)
- return
- }
- // Now we read the data back
- getOpts := minio.GetObjectOptions{}
- getOpts.SetRange(0, 5*1024*1024-1)
- r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject call failed", err)
- return
- }
- getBuf := make([]byte, 5*1024*1024)
- _, err = readFull(r, getBuf)
- if err != nil {
- logError(testName, function, args, startTime, "", "Read buffer failed", err)
- return
- }
- if !bytes.Equal(getBuf, buf) {
- logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err)
- return
- }
- getOpts.SetRange(5*1024*1024, 0)
- r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts)
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject call failed", err)
- return
- }
- getBuf = make([]byte, 5*1024*1024+1)
- _, err = readFull(r, getBuf)
- if err != nil {
- logError(testName, function, args, startTime, "", "Read buffer failed", err)
- return
- }
- if !bytes.Equal(getBuf[:5*1024*1024], buf) {
- logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err)
- return
- }
- if getBuf[5*1024*1024] != buf[0] {
- logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- // Do not need to remove destBucketName its same as bucketName.
- }
- func testUserMetadataCopying() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObject(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- // c.TraceOn(os.Stderr)
- testUserMetadataCopyingWrapper(c)
- }
- func testUserMetadataCopyingWrapper(c *minio.Client) {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObject(destination, source)"
- args := map[string]interface{}{}
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- // Make a new bucket in 'us-east-1' (source bucket).
- err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- fetchMeta := func(object string) (h http.Header) {
- objInfo, err := c.StatObject(context.Background(), bucketName, object, minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat failed", err)
- return
- }
- h = make(http.Header)
- for k, vs := range objInfo.Metadata {
- if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") {
- h.Add(k, vs[0])
- }
- }
- return h
- }
- // 1. create a client encrypted object to copy by uploading
- const srcSize = 1024 * 1024
- buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB
- metadata := make(http.Header)
- metadata.Set("x-amz-meta-myheader", "myvalue")
- m := make(map[string]string)
- m["x-amz-meta-myheader"] = "myvalue"
- _, err = c.PutObject(context.Background(), bucketName, "srcObject",
- bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{UserMetadata: m})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObjectWithMetadata failed", err)
- return
- }
- if !reflect.DeepEqual(metadata, fetchMeta("srcObject")) {
- logError(testName, function, args, startTime, "", "Metadata match failed", err)
- return
- }
- // 2. create source
- src := minio.CopySrcOptions{
- Bucket: bucketName,
- Object: "srcObject",
- }
- // 2.1 create destination with metadata set
- dst1 := minio.CopyDestOptions{
- Bucket: bucketName,
- Object: "dstObject-1",
- UserMetadata: map[string]string{"notmyheader": "notmyvalue"},
- ReplaceMetadata: true,
- }
- // 3. Check that copying to an object with metadata set resets
- // the headers on the copy.
- args["source"] = src
- args["destination"] = dst1
- _, err = c.CopyObject(context.Background(), dst1, src)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObject failed", err)
- return
- }
- expectedHeaders := make(http.Header)
- expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue")
- if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-1")) {
- logError(testName, function, args, startTime, "", "Metadata match failed", err)
- return
- }
- // 4. create destination with no metadata set and same source
- dst2 := minio.CopyDestOptions{
- Bucket: bucketName,
- Object: "dstObject-2",
- }
- // 5. Check that copying to an object with no metadata set,
- // copies metadata.
- args["source"] = src
- args["destination"] = dst2
- _, err = c.CopyObject(context.Background(), dst2, src)
- if err != nil {
- logError(testName, function, args, startTime, "", "CopyObject failed", err)
- return
- }
- expectedHeaders = metadata
- if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-2")) {
- logError(testName, function, args, startTime, "", "Metadata match failed", err)
- return
- }
- // 6. Compose a pair of sources.
- dst3 := minio.CopyDestOptions{
- Bucket: bucketName,
- Object: "dstObject-3",
- ReplaceMetadata: true,
- }
- function = "ComposeObject(destination, sources)"
- args["source"] = []minio.CopySrcOptions{src, src}
- args["destination"] = dst3
- _, err = c.ComposeObject(context.Background(), dst3, src, src)
- if err != nil {
- logError(testName, function, args, startTime, "", "ComposeObject failed", err)
- return
- }
- // Check that no headers are copied in this case
- if !reflect.DeepEqual(make(http.Header), fetchMeta("dstObject-3")) {
- logError(testName, function, args, startTime, "", "Metadata match failed", err)
- return
- }
- // 7. Compose a pair of sources with dest user metadata set.
- dst4 := minio.CopyDestOptions{
- Bucket: bucketName,
- Object: "dstObject-4",
- UserMetadata: map[string]string{"notmyheader": "notmyvalue"},
- ReplaceMetadata: true,
- }
- function = "ComposeObject(destination, sources)"
- args["source"] = []minio.CopySrcOptions{src, src}
- args["destination"] = dst4
- _, err = c.ComposeObject(context.Background(), dst4, src, src)
- if err != nil {
- logError(testName, function, args, startTime, "", "ComposeObject failed", err)
- return
- }
- // Check that no headers are copied in this case
- expectedHeaders = make(http.Header)
- expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue")
- if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-4")) {
- logError(testName, function, args, startTime, "", "Metadata match failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- func testUserMetadataCopyingV2() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "CopyObject(destination, source)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
- return
- }
- // c.TraceOn(os.Stderr)
- testUserMetadataCopyingWrapper(c)
- }
- func testStorageClassMetadataPutObject() {
- // initialize logging params
- startTime := time.Now()
- function := "testStorageClassMetadataPutObject()"
- args := map[string]interface{}{}
- testName := getFuncName()
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
- return
- }
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
- // Make a new bucket in 'us-east-1' (source bucket).
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- fetchMeta := func(object string) (h http.Header) {
- objInfo, err := c.StatObject(context.Background(), bucketName, object, minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat failed", err)
- return
- }
- h = make(http.Header)
- for k, vs := range objInfo.Metadata {
- if strings.HasPrefix(strings.ToLower(k), "x-amz-storage-class") {
- for _, v := range vs {
- h.Add(k, v)
- }
- }
- }
- return h
- }
- metadata := make(http.Header)
- metadata.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
- emptyMetadata := make(http.Header)
- const srcSize = 1024 * 1024
- buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 1MiB
- _, err = c.PutObject(context.Background(), bucketName, "srcObjectRRSClass",
- bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "REDUCED_REDUNDANCY"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- // Get the returned metadata
- returnedMeta := fetchMeta("srcObjectRRSClass")
- // The response metada should either be equal to metadata (with REDUCED_REDUNDANCY) or emptyMetadata (in case of gateways)
- if !reflect.DeepEqual(metadata, returnedMeta) && !reflect.DeepEqual(emptyMetadata, returnedMeta) {
- logError(testName, function, args, startTime, "", "Metadata match failed", err)
- return
- }
- metadata = make(http.Header)
- metadata.Set("x-amz-storage-class", "STANDARD")
- _, err = c.PutObject(context.Background(), bucketName, "srcObjectSSClass",
- bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "STANDARD"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- if reflect.DeepEqual(metadata, fetchMeta("srcObjectSSClass")) {
- logError(testName, function, args, startTime, "", "Metadata verification failed, STANDARD storage class should not be a part of response metadata", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- func testStorageClassInvalidMetadataPutObject() {
- // initialize logging params
- startTime := time.Now()
- function := "testStorageClassInvalidMetadataPutObject()"
- args := map[string]interface{}{}
- testName := getFuncName()
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
- return
- }
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
- // Make a new bucket in 'us-east-1' (source bucket).
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- const srcSize = 1024 * 1024
- buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 1MiB
- _, err = c.PutObject(context.Background(), bucketName, "srcObjectRRSClass",
- bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "INVALID_STORAGE_CLASS"})
- if err == nil {
- logError(testName, function, args, startTime, "", "PutObject with invalid storage class passed, was expected to fail", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- func testStorageClassMetadataCopyObject() {
- // initialize logging params
- startTime := time.Now()
- function := "testStorageClassMetadataCopyObject()"
- args := map[string]interface{}{}
- testName := getFuncName()
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
- return
- }
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
- // Make a new bucket in 'us-east-1' (source bucket).
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- fetchMeta := func(object string) (h http.Header) {
- objInfo, err := c.StatObject(context.Background(), bucketName, object, minio.StatObjectOptions{})
- args["bucket"] = bucketName
- args["object"] = object
- if err != nil {
- logError(testName, function, args, startTime, "", "Stat failed", err)
- return
- }
- h = make(http.Header)
- for k, vs := range objInfo.Metadata {
- if strings.HasPrefix(strings.ToLower(k), "x-amz-storage-class") {
- for _, v := range vs {
- h.Add(k, v)
- }
- }
- }
- return h
- }
- metadata := make(http.Header)
- metadata.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
- emptyMetadata := make(http.Header)
- const srcSize = 1024 * 1024
- buf := bytes.Repeat([]byte("abcde"), srcSize)
- // Put an object with RRS Storage class
- _, err = c.PutObject(context.Background(), bucketName, "srcObjectRRSClass",
- bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "REDUCED_REDUNDANCY"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- // Make server side copy of object uploaded in previous step
- src := minio.CopySrcOptions{
- Bucket: bucketName,
- Object: "srcObjectRRSClass",
- }
- dst := minio.CopyDestOptions{
- Bucket: bucketName,
- Object: "srcObjectRRSClassCopy",
- }
- if _, err = c.CopyObject(context.Background(), dst, src); err != nil {
- logError(testName, function, args, startTime, "", "CopyObject failed on RRS", err)
- return
- }
- // Get the returned metadata
- returnedMeta := fetchMeta("srcObjectRRSClassCopy")
- // The response metada should either be equal to metadata (with REDUCED_REDUNDANCY) or emptyMetadata (in case of gateways)
- if !reflect.DeepEqual(metadata, returnedMeta) && !reflect.DeepEqual(emptyMetadata, returnedMeta) {
- logError(testName, function, args, startTime, "", "Metadata match failed", err)
- return
- }
- metadata = make(http.Header)
- metadata.Set("x-amz-storage-class", "STANDARD")
- // Put an object with Standard Storage class
- _, err = c.PutObject(context.Background(), bucketName, "srcObjectSSClass",
- bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "STANDARD"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- // Make server side copy of object uploaded in previous step
- src = minio.CopySrcOptions{
- Bucket: bucketName,
- Object: "srcObjectSSClass",
- }
- dst = minio.CopyDestOptions{
- Bucket: bucketName,
- Object: "srcObjectSSClassCopy",
- }
- if _, err = c.CopyObject(context.Background(), dst, src); err != nil {
- logError(testName, function, args, startTime, "", "CopyObject failed on SS", err)
- return
- }
- // Fetch the meta data of copied object
- if reflect.DeepEqual(metadata, fetchMeta("srcObjectSSClassCopy")) {
- logError(testName, function, args, startTime, "", "Metadata verification failed, STANDARD storage class should not be a part of response metadata", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test put object with size -1 byte object.
- func testPutObjectNoLengthV2() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "PutObject(bucketName, objectName, reader, size, opts)"
- args := map[string]interface{}{
- "bucketName": "",
- "objectName": "",
- "size": -1,
- "opts": "",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- objectName := bucketName + "unique"
- args["objectName"] = objectName
- bufSize := dataFileMap["datafile-129-MB"]
- var reader = getDataReader("datafile-129-MB")
- defer reader.Close()
- args["size"] = bufSize
- // Upload an object.
- _, err = c.PutObject(context.Background(), bucketName, objectName, reader, -1, minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err)
- return
- }
- st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject failed", err)
- return
- }
- if st.Size != int64(bufSize) {
- logError(testName, function, args, startTime, "", "Expected upload object size "+string(bufSize)+" got "+string(st.Size), err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test put objects of unknown size.
- func testPutObjectsUnknownV2() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "PutObject(bucketName, objectName, reader,size,opts)"
- args := map[string]interface{}{
- "bucketName": "",
- "objectName": "",
- "size": "",
- "opts": "",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Issues are revealed by trying to upload multiple files of unknown size
- // sequentially (on 4GB machines)
- for i := 1; i <= 4; i++ {
- // Simulate that we could be receiving byte slices of data that we want
- // to upload as a file
- rpipe, wpipe := io.Pipe()
- defer rpipe.Close()
- go func() {
- b := []byte("test")
- wpipe.Write(b)
- wpipe.Close()
- }()
- // Upload the object.
- objectName := fmt.Sprintf("%sunique%d", bucketName, i)
- args["objectName"] = objectName
- ui, err := c.PutObject(context.Background(), bucketName, objectName, rpipe, -1, minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err)
- return
- }
- if ui.Size != 4 {
- logError(testName, function, args, startTime, "", "Expected upload object size "+string(4)+" got "+string(ui.Size), nil)
- return
- }
- st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObjectStreaming failed", err)
- return
- }
- if st.Size != int64(4) {
- logError(testName, function, args, startTime, "", "Expected upload object size "+string(4)+" got "+string(st.Size), err)
- return
- }
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test put object with 0 byte object.
- func testPutObject0ByteV2() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "PutObject(bucketName, objectName, reader, size, opts)"
- args := map[string]interface{}{
- "bucketName": "",
- "objectName": "",
- "size": 0,
- "opts": "",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- objectName := bucketName + "unique"
- args["objectName"] = objectName
- args["opts"] = minio.PutObjectOptions{}
- // Upload an object.
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader([]byte("")), 0, minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err)
- return
- }
- st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObjectWithSize failed", err)
- return
- }
- if st.Size != 0 {
- logError(testName, function, args, startTime, "", "Expected upload object size 0 but got "+string(st.Size), err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test expected error cases
- func testComposeObjectErrorCases() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "ComposeObject(destination, sourceList)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- testComposeObjectErrorCasesWrapper(c)
- }
- // Test concatenating multiple 10K objects V4
- func testCompose10KSources() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "ComposeObject(destination, sourceList)"
- args := map[string]interface{}{}
- // Instantiate new minio client object
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
- return
- }
- testComposeMultipleSources(c)
- }
- // Tests comprehensive list of all methods.
- func testFunctionalV2() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "testFunctionalV2()"
- functionAll := ""
- args := map[string]interface{}{}
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
- return
- }
- // Enable to debug
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- location := "us-east-1"
- // Make a new bucket.
- function = "MakeBucket(bucketName, location)"
- functionAll = "MakeBucket(bucketName, location)"
- args = map[string]interface{}{
- "bucketName": bucketName,
- "location": location,
- }
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: location})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- // Generate a random file name.
- fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- file, err := os.Create(fileName)
- if err != nil {
- logError(testName, function, args, startTime, "", "file create failed", err)
- return
- }
- for i := 0; i < 3; i++ {
- buf := make([]byte, rand.Intn(1<<19))
- _, err = file.Write(buf)
- if err != nil {
- logError(testName, function, args, startTime, "", "file write failed", err)
- return
- }
- }
- file.Close()
- // Verify if bucket exits and you have access.
- var exists bool
- function = "BucketExists(bucketName)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- }
- exists, err = c.BucketExists(context.Background(), bucketName)
- if err != nil {
- logError(testName, function, args, startTime, "", "BucketExists failed", err)
- return
- }
- if !exists {
- logError(testName, function, args, startTime, "", "Could not find existing bucket "+bucketName, err)
- return
- }
- // Make the bucket 'public read/write'.
- function = "SetBucketPolicy(bucketName, bucketPolicy)"
- functionAll += ", " + function
- readWritePolicy := `{"Version": "2012-10-17","Statement": [{"Action": ["s3:ListBucketMultipartUploads", "s3:ListBucket"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::` + bucketName + `"],"Sid": ""}]}`
- args = map[string]interface{}{
- "bucketName": bucketName,
- "bucketPolicy": readWritePolicy,
- }
- err = c.SetBucketPolicy(context.Background(), bucketName, readWritePolicy)
- if err != nil {
- logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err)
- return
- }
- // List all buckets.
- function = "ListBuckets()"
- functionAll += ", " + function
- args = nil
- buckets, err := c.ListBuckets(context.Background())
- if len(buckets) == 0 {
- logError(testName, function, args, startTime, "", "List buckets cannot be empty", err)
- return
- }
- if err != nil {
- logError(testName, function, args, startTime, "", "ListBuckets failed", err)
- return
- }
- // Verify if previously created bucket is listed in list buckets.
- bucketFound := false
- for _, bucket := range buckets {
- if bucket.Name == bucketName {
- bucketFound = true
- }
- }
- // If bucket not found error out.
- if !bucketFound {
- logError(testName, function, args, startTime, "", "Bucket "+bucketName+"not found", err)
- return
- }
- objectName := bucketName + "unique"
- // Generate data
- buf := bytes.Repeat([]byte("n"), rand.Intn(1<<19))
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": objectName,
- "contentType": "",
- }
- _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject failed", err)
- return
- }
- if st.Size != int64(len(buf)) {
- logError(testName, function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(st.Size), err)
- return
- }
- objectNameNoLength := objectName + "-nolength"
- args["objectName"] = objectNameNoLength
- _, err = c.PutObject(context.Background(), bucketName, objectNameNoLength, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- st, err = c.StatObject(context.Background(), bucketName, objectNameNoLength, minio.StatObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "StatObject failed", err)
- return
- }
- if st.Size != int64(len(buf)) {
- logError(testName, function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(st.Size), err)
- return
- }
- // Instantiate a done channel to close all listing.
- doneCh := make(chan struct{})
- defer close(doneCh)
- objFound := false
- isRecursive := true // Recursive is true.
- function = "ListObjects(bucketName, objectName, isRecursive, doneCh)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": objectName,
- "isRecursive": isRecursive,
- }
- for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Prefix: objectName, Recursive: isRecursive}) {
- if obj.Key == objectName {
- objFound = true
- break
- }
- }
- if !objFound {
- logError(testName, function, args, startTime, "", "Could not find existing object "+objectName, err)
- return
- }
- incompObjNotFound := true
- function = "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": objectName,
- "isRecursive": isRecursive,
- }
- for objIncompl := range c.ListIncompleteUploads(context.Background(), bucketName, objectName, isRecursive) {
- if objIncompl.Key != "" {
- incompObjNotFound = false
- break
- }
- }
- if !incompObjNotFound {
- logError(testName, function, args, startTime, "", "Unexpected dangling incomplete upload found", err)
- return
- }
- function = "GetObject(bucketName, objectName)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": objectName,
- }
- newReader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- newReadBytes, err := ioutil.ReadAll(newReader)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAll failed", err)
- return
- }
- newReader.Close()
- if !bytes.Equal(newReadBytes, buf) {
- logError(testName, function, args, startTime, "", "Bytes mismatch", err)
- return
- }
- function = "FGetObject(bucketName, objectName, fileName)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": objectName,
- "fileName": fileName + "-f",
- }
- err = c.FGetObject(context.Background(), bucketName, objectName, fileName+"-f", minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "FgetObject failed", err)
- return
- }
- // Generate presigned HEAD object url.
- function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": objectName,
- "expires": 3600 * time.Second,
- }
- presignedHeadURL, err := c.PresignedHeadObject(context.Background(), bucketName, objectName, 3600*time.Second, nil)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedHeadObject failed", err)
- return
- }
- transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS)))
- if err != nil {
- logError(testName, function, args, startTime, "", "DefaultTransport failed", err)
- return
- }
- httpClient := &http.Client{
- // Setting a sensible time out of 30secs to wait for response
- // headers. Request is pro-actively canceled after 30secs
- // with no response.
- Timeout: 30 * time.Second,
- Transport: transport,
- }
- req, err := http.NewRequest(http.MethodHead, presignedHeadURL.String(), nil)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedHeadObject URL head request failed", err)
- return
- }
- // Verify if presigned url works.
- resp, err := httpClient.Do(req)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedHeadObject URL head request failed", err)
- return
- }
- if resp.StatusCode != http.StatusOK {
- logError(testName, function, args, startTime, "", "PresignedHeadObject URL returns status "+string(resp.StatusCode), err)
- return
- }
- if resp.Header.Get("ETag") == "" {
- logError(testName, function, args, startTime, "", "Got empty ETag", err)
- return
- }
- resp.Body.Close()
- // Generate presigned GET object url.
- function = "PresignedGetObject(bucketName, objectName, expires, reqParams)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": objectName,
- "expires": 3600 * time.Second,
- }
- presignedGetURL, err := c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, nil)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedGetObject failed", err)
- return
- }
- // Verify if presigned url works.
- req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err)
- return
- }
- resp, err = httpClient.Do(req)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
- return
- }
- if resp.StatusCode != http.StatusOK {
- logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err)
- return
- }
- newPresignedBytes, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAll failed", err)
- return
- }
- resp.Body.Close()
- if !bytes.Equal(newPresignedBytes, buf) {
- logError(testName, function, args, startTime, "", "Bytes mismatch", err)
- return
- }
- // Set request parameters.
- reqParams := make(url.Values)
- reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"")
- // Generate presigned GET object url.
- args["reqParams"] = reqParams
- presignedGetURL, err = c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, reqParams)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedGetObject failed", err)
- return
- }
- // Verify if presigned url works.
- req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err)
- return
- }
- resp, err = httpClient.Do(req)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
- return
- }
- if resp.StatusCode != http.StatusOK {
- logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err)
- return
- }
- newPresignedBytes, err = ioutil.ReadAll(resp.Body)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAll failed", err)
- return
- }
- if !bytes.Equal(newPresignedBytes, buf) {
- logError(testName, function, args, startTime, "", "Bytes mismatch", err)
- return
- }
- // Verify content disposition.
- if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" {
- logError(testName, function, args, startTime, "", "wrong Content-Disposition received ", err)
- return
- }
- function = "PresignedPutObject(bucketName, objectName, expires)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": objectName + "-presigned",
- "expires": 3600 * time.Second,
- }
- presignedPutURL, err := c.PresignedPutObject(context.Background(), bucketName, objectName+"-presigned", 3600*time.Second)
- if err != nil {
- logError(testName, function, args, startTime, "", "PresignedPutObject failed", err)
- return
- }
- // Generate data more than 32K
- buf = bytes.Repeat([]byte("1"), rand.Intn(1<<10)+32*1024)
- req, err = http.NewRequest(http.MethodPut, presignedPutURL.String(), bytes.NewReader(buf))
- if err != nil {
- logError(testName, function, args, startTime, "", "HTTP request to PresignedPutObject URL failed", err)
- return
- }
- resp, err = httpClient.Do(req)
- if err != nil {
- logError(testName, function, args, startTime, "", "HTTP request to PresignedPutObject URL failed", err)
- return
- }
- function = "GetObject(bucketName, objectName)"
- functionAll += ", " + function
- args = map[string]interface{}{
- "bucketName": bucketName,
- "objectName": objectName + "-presigned",
- }
- newReader, err = c.GetObject(context.Background(), bucketName, objectName+"-presigned", minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- newReadBytes, err = ioutil.ReadAll(newReader)
- if err != nil {
- logError(testName, function, args, startTime, "", "ReadAll failed", err)
- return
- }
- newReader.Close()
- if !bytes.Equal(newReadBytes, buf) {
- logError(testName, function, args, startTime, "", "Bytes mismatch", err)
- return
- }
- os.Remove(fileName)
- os.Remove(fileName + "-f")
- successLogger(testName, functionAll, args, startTime).Info()
- }
- // Test get object with GetObject with context
- func testGetObjectContext() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "GetObject(ctx, bucketName, objectName)"
- args := map[string]interface{}{
- "ctx": "",
- "bucketName": "",
- "objectName": "",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- bufSize := dataFileMap["datafile-33-kB"]
- var reader = getDataReader("datafile-33-kB")
- defer reader.Close()
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
- args["ctx"] = ctx
- cancel()
- r, err := c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed unexpectedly", err)
- return
- }
- if _, err = r.Stat(); err == nil {
- logError(testName, function, args, startTime, "", "GetObject should fail on short timeout", err)
- return
- }
- r.Close()
- ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
- args["ctx"] = ctx
- defer cancel()
- // Read the data back
- r, err = c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed", err)
- return
- }
- st, err := r.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "object Stat call failed", err)
- return
- }
- if st.Size != int64(bufSize) {
- logError(testName, function, args, startTime, "", "Number of bytes in stat does not match: want "+string(bufSize)+", got"+string(st.Size), err)
- return
- }
- if err := r.Close(); err != nil {
- logError(testName, function, args, startTime, "", "object Close() call failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test get object with FGetObject with a user provided context
- func testFGetObjectContext() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "FGetObject(ctx, bucketName, objectName, fileName)"
- args := map[string]interface{}{
- "ctx": "",
- "bucketName": "",
- "objectName": "",
- "fileName": "",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- bufSize := dataFileMap["datafile-1-MB"]
- var reader = getDataReader("datafile-1-MB")
- defer reader.Close()
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
- args["ctx"] = ctx
- defer cancel()
- fileName := "tempfile-context"
- args["fileName"] = fileName
- // Read the data back
- err = c.FGetObject(ctx, bucketName, objectName, fileName+"-f", minio.GetObjectOptions{})
- if err == nil {
- logError(testName, function, args, startTime, "", "FGetObject should fail on short timeout", err)
- return
- }
- ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
- defer cancel()
- // Read the data back
- err = c.FGetObject(ctx, bucketName, objectName, fileName+"-fcontext", minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "FGetObject with long timeout failed", err)
- return
- }
- if err = os.Remove(fileName + "-fcontext"); err != nil {
- logError(testName, function, args, startTime, "", "Remove file failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test get object with GetObject with a user provided context
- func testGetObjectRanges() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "GetObject(ctx, bucketName, objectName, fileName)"
- args := map[string]interface{}{
- "ctx": "",
- "bucketName": "",
- "objectName": "",
- "fileName": "",
- }
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
- defer cancel()
- rng := rand.NewSource(time.Now().UnixNano())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rng, "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- bufSize := dataFileMap["datafile-129-MB"]
- var reader = getDataReader("datafile-129-MB")
- defer reader.Close()
- // Save the data
- objectName := randString(60, rng, "")
- args["objectName"] = objectName
- _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- // Read the data back
- tests := []struct {
- start int64
- end int64
- }{
- {
- start: 1024,
- end: 1024 + 1<<20,
- },
- {
- start: 20e6,
- end: 20e6 + 10000,
- },
- {
- start: 40e6,
- end: 40e6 + 10000,
- },
- {
- start: 60e6,
- end: 60e6 + 10000,
- },
- {
- start: 80e6,
- end: 80e6 + 10000,
- },
- {
- start: 120e6,
- end: int64(bufSize),
- },
- }
- for _, test := range tests {
- wantRC := getDataReader("datafile-129-MB")
- io.CopyN(ioutil.Discard, wantRC, test.start)
- want := mustCrcReader(io.LimitReader(wantRC, test.end-test.start+1))
- opts := minio.GetObjectOptions{}
- opts.SetRange(test.start, test.end)
- args["opts"] = fmt.Sprintf("%+v", test)
- obj, err := c.GetObject(ctx, bucketName, objectName, opts)
- if err != nil {
- logError(testName, function, args, startTime, "", "FGetObject with long timeout failed", err)
- return
- }
- err = crcMatches(obj, want)
- if err != nil {
- logError(testName, function, args, startTime, "", fmt.Sprintf("GetObject offset %d -> %d", test.start, test.end), err)
- return
- }
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test get object ACLs with GetObjectACL with custom provided context
- func testGetObjectACLContext() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "GetObjectACL(ctx, bucketName, objectName)"
- args := map[string]interface{}{
- "ctx": "",
- "bucketName": "",
- "objectName": "",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // skipping region functional tests for non s3 runs
- if os.Getenv(serverEndpoint) != "s3.amazonaws.com" {
- ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info()
- return
- }
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- bufSize := dataFileMap["datafile-1-MB"]
- var reader = getDataReader("datafile-1-MB")
- defer reader.Close()
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- // Add meta data to add a canned acl
- metaData := map[string]string{
- "X-Amz-Acl": "public-read-write",
- }
- _, err = c.PutObject(context.Background(), bucketName,
- objectName, reader, int64(bufSize),
- minio.PutObjectOptions{
- ContentType: "binary/octet-stream",
- UserMetadata: metaData,
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- args["ctx"] = ctx
- defer cancel()
- // Read the data back
- objectInfo, getObjectACLErr := c.GetObjectACL(ctx, bucketName, objectName)
- if getObjectACLErr != nil {
- logError(testName, function, args, startTime, "", "GetObjectACL failed. ", getObjectACLErr)
- return
- }
- s, ok := objectInfo.Metadata["X-Amz-Acl"]
- if !ok {
- logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Acl\"", nil)
- return
- }
- if len(s) != 1 {
- logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" canned acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil)
- return
- }
- if s[0] != "public-read-write" {
- logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" expected \"public-read-write\" but got"+fmt.Sprintf("%q", s[0]), nil)
- return
- }
- bufSize = dataFileMap["datafile-1-MB"]
- var reader2 = getDataReader("datafile-1-MB")
- defer reader2.Close()
- // Save the data
- objectName = randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- // Add meta data to add a canned acl
- metaData = map[string]string{
- "X-Amz-Grant-Read": "id=fooread@minio.go",
- "X-Amz-Grant-Write": "id=foowrite@minio.go",
- }
- _, err = c.PutObject(context.Background(), bucketName, objectName, reader2, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", UserMetadata: metaData})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject failed", err)
- return
- }
- ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
- args["ctx"] = ctx
- defer cancel()
- // Read the data back
- objectInfo, getObjectACLErr = c.GetObjectACL(ctx, bucketName, objectName)
- if getObjectACLErr == nil {
- logError(testName, function, args, startTime, "", "GetObjectACL fail", getObjectACLErr)
- return
- }
- if len(objectInfo.Metadata) != 3 {
- logError(testName, function, args, startTime, "", "GetObjectACL fail expected \"3\" ACLs but got "+fmt.Sprintf(`"%d"`, len(objectInfo.Metadata)), nil)
- return
- }
- s, ok = objectInfo.Metadata["X-Amz-Grant-Read"]
- if !ok {
- logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Grant-Read\"", nil)
- return
- }
- if len(s) != 1 {
- logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Read\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil)
- return
- }
- if s[0] != "fooread@minio.go" {
- logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Read\" acl expected \"fooread@minio.go\" got "+fmt.Sprintf("%q", s), nil)
- return
- }
- s, ok = objectInfo.Metadata["X-Amz-Grant-Write"]
- if !ok {
- logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Grant-Write\"", nil)
- return
- }
- if len(s) != 1 {
- logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Write\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil)
- return
- }
- if s[0] != "foowrite@minio.go" {
- logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Write\" acl expected \"foowrite@minio.go\" got "+fmt.Sprintf("%q", s), nil)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test validates putObject with context to see if request cancellation is honored for V2.
- func testPutObjectContextV2() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "PutObject(ctx, bucketName, objectName, reader, size, opts)"
- args := map[string]interface{}{
- "ctx": "",
- "bucketName": "",
- "objectName": "",
- "size": "",
- "opts": "",
- }
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Make a new bucket.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- bufSize := dataFileMap["datatfile-33-kB"]
- var reader = getDataReader("datafile-33-kB")
- defer reader.Close()
- objectName := fmt.Sprintf("test-file-%v", rand.Uint32())
- args["objectName"] = objectName
- ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
- args["ctx"] = ctx
- args["size"] = bufSize
- defer cancel()
- _, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject with short timeout failed", err)
- return
- }
- ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
- args["ctx"] = ctx
- defer cancel()
- reader = getDataReader("datafile-33-kB")
- defer reader.Close()
- _, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject with long timeout failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test get object with GetObject with custom context
- func testGetObjectContextV2() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "GetObject(ctx, bucketName, objectName)"
- args := map[string]interface{}{
- "ctx": "",
- "bucketName": "",
- "objectName": "",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- bufSize := dataFileMap["datafile-33-kB"]
- var reader = getDataReader("datafile-33-kB")
- defer reader.Close()
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject call failed", err)
- return
- }
- ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
- args["ctx"] = ctx
- cancel()
- r, err := c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject failed unexpectedly", err)
- return
- }
- if _, err = r.Stat(); err == nil {
- logError(testName, function, args, startTime, "", "GetObject should fail on short timeout", err)
- return
- }
- r.Close()
- ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
- defer cancel()
- // Read the data back
- r, err = c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "GetObject shouldn't fail on longer timeout", err)
- return
- }
- st, err := r.Stat()
- if err != nil {
- logError(testName, function, args, startTime, "", "object Stat call failed", err)
- return
- }
- if st.Size != int64(bufSize) {
- logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(bufSize)+" got "+string(st.Size), err)
- return
- }
- if err := r.Close(); err != nil {
- logError(testName, function, args, startTime, "", " object Close() call failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test get object with FGetObject with custom context
- func testFGetObjectContextV2() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "FGetObject(ctx, bucketName, objectName,fileName)"
- args := map[string]interface{}{
- "ctx": "",
- "bucketName": "",
- "objectName": "",
- "fileName": "",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket call failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- bufSize := dataFileMap["datatfile-1-MB"]
- var reader = getDataReader("datafile-1-MB")
- defer reader.Close()
- // Save the data
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
- if err != nil {
- logError(testName, function, args, startTime, "", "PutObject call failed", err)
- return
- }
- ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
- args["ctx"] = ctx
- defer cancel()
- fileName := "tempfile-context"
- args["fileName"] = fileName
- // Read the data back
- err = c.FGetObject(ctx, bucketName, objectName, fileName+"-f", minio.GetObjectOptions{})
- if err == nil {
- logError(testName, function, args, startTime, "", "FGetObject should fail on short timeout", err)
- return
- }
- ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour)
- defer cancel()
- // Read the data back
- err = c.FGetObject(ctx, bucketName, objectName, fileName+"-fcontext", minio.GetObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "FGetObject call shouldn't fail on long timeout", err)
- return
- }
- if err = os.Remove(fileName + "-fcontext"); err != nil {
- logError(testName, function, args, startTime, "", "Remove file failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Test list object v1 and V2
- func testListObjects() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "ListObjects(bucketName, objectPrefix, recursive, doneCh)"
- args := map[string]interface{}{
- "bucketName": "",
- "objectPrefix": "",
- "recursive": "true",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- defer cleanupBucket(bucketName, c)
- testObjects := []struct {
- name string
- storageClass string
- }{
- // Special characters
- {"foo bar", "STANDARD"},
- {"foo-%", "STANDARD"},
- {"random-object-1", "STANDARD"},
- {"random-object-2", "REDUCED_REDUNDANCY"},
- }
- for i, object := range testObjects {
- bufSize := dataFileMap["datafile-33-kB"]
- var reader = getDataReader("datafile-33-kB")
- defer reader.Close()
- _, err = c.PutObject(context.Background(), bucketName, object.name, reader, int64(bufSize),
- minio.PutObjectOptions{ContentType: "binary/octet-stream", StorageClass: object.storageClass})
- if err != nil {
- logError(testName, function, args, startTime, "", fmt.Sprintf("PutObject %d call failed", i+1), err)
- return
- }
- }
- testList := func(listFn func(context.Context, string, minio.ListObjectsOptions) <-chan minio.ObjectInfo, bucket string, opts minio.ListObjectsOptions) {
- var objCursor int
- // check for object name and storage-class from listing object result
- for objInfo := range listFn(context.Background(), bucket, opts) {
- if objInfo.Err != nil {
- logError(testName, function, args, startTime, "", "ListObjects failed unexpectedly", err)
- return
- }
- if objInfo.Key != testObjects[objCursor].name {
- logError(testName, function, args, startTime, "", "ListObjects does not return expected object name", err)
- return
- }
- if objInfo.StorageClass != testObjects[objCursor].storageClass {
- // Ignored as Gateways (Azure/GCS etc) wont return storage class
- ignoredLog(testName, function, args, startTime, "ListObjects doesn't return expected storage class").Info()
- }
- objCursor++
- }
- if objCursor != len(testObjects) {
- logError(testName, function, args, startTime, "", "ListObjects returned unexpected number of items", errors.New(""))
- return
- }
- }
- testList(c.ListObjects, bucketName, minio.ListObjectsOptions{Recursive: true, UseV1: true})
- testList(c.ListObjects, bucketName, minio.ListObjectsOptions{Recursive: true})
- testList(c.ListObjects, bucketName, minio.ListObjectsOptions{Recursive: true, WithMetadata: true})
- successLogger(testName, function, args, startTime).Info()
- }
- // Test deleting multiple objects with object retention set in Governance mode
- func testRemoveObjects() {
- // initialize logging params
- startTime := time.Now()
- testName := getFuncName()
- function := "RemoveObjects(bucketName, objectsCh, opts)"
- args := map[string]interface{}{
- "bucketName": "",
- "objectPrefix": "",
- "recursive": "true",
- }
- // Seed random based on current time.
- rand.Seed(time.Now().Unix())
- // Instantiate new minio client object.
- c, err := minio.New(os.Getenv(serverEndpoint),
- &minio.Options{
- Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
- Secure: mustParseBool(os.Getenv(enableHTTPS)),
- })
- if err != nil {
- logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err)
- return
- }
- // Enable tracing, write to stderr.
- // c.TraceOn(os.Stderr)
- // Set user agent.
- c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
- // Generate a new random bucket name.
- bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
- args["bucketName"] = bucketName
- objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
- args["objectName"] = objectName
- // Make a new bucket.
- err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
- if err != nil {
- logError(testName, function, args, startTime, "", "MakeBucket failed", err)
- return
- }
- bufSize := dataFileMap["datafile-129-MB"]
- var reader = getDataReader("datafile-129-MB")
- defer reader.Close()
- _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "Error uploading object", err)
- }
- // Replace with smaller...
- bufSize = dataFileMap["datafile-10-kB"]
- reader = getDataReader("datafile-10-kB")
- defer reader.Close()
- _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{})
- if err != nil {
- logError(testName, function, args, startTime, "", "Error uploading object", err)
- }
- t := time.Date(2030, time.April, 25, 14, 0, 0, 0, time.UTC)
- m := minio.RetentionMode(minio.Governance)
- opts := minio.PutObjectRetentionOptions{
- GovernanceBypass: false,
- RetainUntilDate: &t,
- Mode: &m,
- }
- err = c.PutObjectRetention(context.Background(), bucketName, objectName, opts)
- if err != nil {
- log.Fatalln(err)
- }
- objectsCh := make(chan minio.ObjectInfo)
- // Send object names that are needed to be removed to objectsCh
- go func() {
- defer close(objectsCh)
- // List all objects from a bucket-name with a matching prefix.
- for object := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Recursive: true}) {
- if object.Err != nil {
- log.Fatalln(object.Err)
- }
- objectsCh <- object
- }
- }()
- for rErr := range c.RemoveObjects(context.Background(), bucketName, objectsCh, minio.RemoveObjectsOptions{}) {
- // Error is expected here because Retention is set on the object
- // and RemoveObjects is called without Bypass Governance
- if rErr.Err == nil {
- logError(testName, function, args, startTime, "", "Expected error during deletion", nil)
- return
- }
- }
- objectsCh1 := make(chan minio.ObjectInfo)
- // Send object names that are needed to be removed to objectsCh
- go func() {
- defer close(objectsCh1)
- // List all objects from a bucket-name with a matching prefix.
- for object := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Recursive: true}) {
- if object.Err != nil {
- log.Fatalln(object.Err)
- }
- objectsCh1 <- object
- }
- }()
- opts1 := minio.RemoveObjectsOptions{
- GovernanceBypass: true,
- }
- for rErr := range c.RemoveObjects(context.Background(), bucketName, objectsCh1, opts1) {
- // Error is not expected here because Retention is set on the object
- // and RemoveObjects is called with Bypass Governance
- logError(testName, function, args, startTime, "", "Error detected during deletion", rErr.Err)
- return
- }
- // Delete all objects and buckets
- if err = cleanupVersionedBucket(bucketName, c); err != nil {
- logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
- return
- }
- successLogger(testName, function, args, startTime).Info()
- }
- // Convert string to bool and always return false if any error
- func mustParseBool(str string) bool {
- b, err := strconv.ParseBool(str)
- if err != nil {
- return false
- }
- return b
- }
- func main() {
- // Output to stdout instead of the default stderr
- log.SetOutput(os.Stdout)
- // create custom formatter
- mintFormatter := mintJSONFormatter{}
- // set custom formatter
- log.SetFormatter(&mintFormatter)
- // log Info or above -- success cases are Info level, failures are Fatal level
- log.SetLevel(log.InfoLevel)
- tls := mustParseBool(os.Getenv(enableHTTPS))
- kms := mustParseBool(os.Getenv(enableKMS))
- if os.Getenv(enableKMS) == "" {
- // Default to KMS tests.
- kms = true
- }
- // execute tests
- if isFullMode() {
- testMakeBucketErrorV2()
- testGetObjectClosedTwiceV2()
- testFPutObjectV2()
- testMakeBucketRegionsV2()
- testGetObjectReadSeekFunctionalV2()
- testGetObjectReadAtFunctionalV2()
- testGetObjectRanges()
- testCopyObjectV2()
- testFunctionalV2()
- testComposeObjectErrorCasesV2()
- testCompose10KSourcesV2()
- testUserMetadataCopyingV2()
- testPutObject0ByteV2()
- testPutObjectNoLengthV2()
- testPutObjectsUnknownV2()
- testGetObjectContextV2()
- testFPutObjectContextV2()
- testFGetObjectContextV2()
- testPutObjectContextV2()
- testPutObjectWithVersioning()
- testMakeBucketError()
- testMakeBucketRegions()
- testPutObjectWithMetadata()
- testPutObjectReadAt()
- testPutObjectStreaming()
- testGetObjectSeekEnd()
- testGetObjectClosedTwice()
- testRemoveMultipleObjects()
- testFPutObjectMultipart()
- testFPutObject()
- testGetObjectReadSeekFunctional()
- testGetObjectReadAtFunctional()
- testGetObjectReadAtWhenEOFWasReached()
- testPresignedPostPolicy()
- testCopyObject()
- testComposeObjectErrorCases()
- testCompose10KSources()
- testUserMetadataCopying()
- testBucketNotification()
- testFunctional()
- testGetObjectModified()
- testPutObjectUploadSeekedObject()
- testGetObjectContext()
- testFPutObjectContext()
- testFGetObjectContext()
- testGetObjectACLContext()
- testPutObjectContext()
- testStorageClassMetadataPutObject()
- testStorageClassInvalidMetadataPutObject()
- testStorageClassMetadataCopyObject()
- testPutObjectWithContentLanguage()
- testListObjects()
- testRemoveObjects()
- testListObjectVersions()
- testStatObjectWithVersioning()
- testGetObjectWithVersioning()
- testCopyObjectWithVersioning()
- testConcurrentCopyObjectWithVersioning()
- testComposeObjectWithVersioning()
- testRemoveObjectWithVersioning()
- testRemoveObjectsWithVersioning()
- testObjectTaggingWithVersioning()
- // SSE-C tests will only work over TLS connection.
- if tls {
- testSSECEncryptionPutGet()
- testSSECEncryptionFPut()
- testSSECEncryptedGetObjectReadAtFunctional()
- testSSECEncryptedGetObjectReadSeekFunctional()
- testEncryptedCopyObjectV2()
- testEncryptedSSECToSSECCopyObject()
- testEncryptedSSECToUnencryptedCopyObject()
- testUnencryptedToSSECCopyObject()
- testUnencryptedToUnencryptedCopyObject()
- testEncryptedEmptyObject()
- testDecryptedCopyObject()
- testSSECEncryptedToSSECCopyObjectPart()
- testSSECMultipartEncryptedToSSECCopyObjectPart()
- testSSECEncryptedToUnencryptedCopyPart()
- testUnencryptedToSSECCopyObjectPart()
- testUnencryptedToUnencryptedCopyPart()
- testEncryptedSSECToSSES3CopyObject()
- testEncryptedSSES3ToSSECCopyObject()
- testSSECEncryptedToSSES3CopyObjectPart()
- testSSES3EncryptedToSSECCopyObjectPart()
- }
- // KMS tests
- if kms {
- testSSES3EncryptionPutGet()
- testSSES3EncryptionFPut()
- testSSES3EncryptedGetObjectReadAtFunctional()
- testSSES3EncryptedGetObjectReadSeekFunctional()
- testEncryptedSSES3ToSSES3CopyObject()
- testEncryptedSSES3ToUnencryptedCopyObject()
- testUnencryptedToSSES3CopyObject()
- testUnencryptedToSSES3CopyObjectPart()
- testSSES3EncryptedToUnencryptedCopyPart()
- testSSES3EncryptedToSSES3CopyObjectPart()
- }
- } else {
- testFunctional()
- testFunctionalV2()
- }
- }
|