diff --git a/chirps/chirps/settings.py b/chirps/chirps/settings.py index d5a17816..7aad65d1 100644 --- a/chirps/chirps/settings.py +++ b/chirps/chirps/settings.py @@ -54,6 +54,7 @@ 'target', 'account', 'plan', + 'embedding', ] MIDDLEWARE = [ diff --git a/chirps/chirps/urls.py b/chirps/chirps/urls.py index 7b3f5a5b..927e1f1a 100644 --- a/chirps/chirps/urls.py +++ b/chirps/chirps/urls.py @@ -20,9 +20,10 @@ urlpatterns = [ path('', include('base_app.urls')), - path('target/', include('target.urls')), - path('scan/', include('scan.urls')), - path('plan/', include('plan.urls')), - path('admin/', admin.site.urls), path('account/', include('account.urls')), + path('admin/', admin.site.urls), + path('embedding/', include('embedding.urls')), + path('plan/', include('plan.urls')), + path('scan/', include('scan.urls')), + path('target/', include('target.urls')), ] diff --git a/chirps/embedding/__init__.py b/chirps/embedding/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/chirps/embedding/admin.py b/chirps/embedding/admin.py new file mode 100644 index 00000000..539f54a8 --- /dev/null +++ b/chirps/embedding/admin.py @@ -0,0 +1,6 @@ +"""Register Embedding model with admin site.""" +from django.contrib import admin + +from .models import Embedding + +admin.register(Embedding) diff --git a/chirps/embedding/apps.py b/chirps/embedding/apps.py new file mode 100644 index 00000000..68d52c2b --- /dev/null +++ b/chirps/embedding/apps.py @@ -0,0 +1,9 @@ +"""Embedding application configuration options.""" +from django.apps import AppConfig + + +class EmbeddingConfig(AppConfig): + """Configuration for the embedding app.""" + + default_auto_field = 'django.db.models.BigAutoField' + name = 'embedding' diff --git a/chirps/embedding/fixtures/__init__.py b/chirps/embedding/fixtures/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/chirps/embedding/fixtures/embedding/__init__.py b/chirps/embedding/fixtures/embedding/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/chirps/embedding/fixtures/embedding/openai_embedding_create_mock.json b/chirps/embedding/fixtures/embedding/openai_embedding_create_mock.json new file mode 100644 index 00000000..ce859935 --- /dev/null +++ b/chirps/embedding/fixtures/embedding/openai_embedding_create_mock.json @@ -0,0 +1,1552 @@ +{ + "object": "list", + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + -0.016634132713079453, + -0.015486475080251694, + -0.01258276030421257, + -0.022026745602488518, + -0.013592147268354893, + 0.006813357584178448, + -0.022593660280108452, + -0.02269045077264309, + -0.007245457731187344, + -0.008593610487878323, + 0.024308234453201294, + -0.003332357620820403, + 0.0073906434699893, + -0.003432604717090726, + -0.023160576820373535, + -0.012714119628071785, + 0.028954176232218742, + 0.0032580362167209387, + 0.00369532173499465, + -0.0024042059667408466, + -0.020561059936881065, + 0.011144731193780899, + 0.0040824837051332, + 0.0012850662460550666, + -0.027861827984452248, + 0.008752623572945595, + 0.006557554006576538, + -0.018072163686156273, + -0.00824793055653572, + -0.003190628718584776, + 0.03440209850668907, + -0.024086998775601387, + -0.022400079295039177, + 0.005237055476754904, + -0.02776503749191761, + -0.023907246068120003, + 0.0014190173242241144, + -0.00940941646695137, + 0.016053389757871628, + -0.013778814114630222, + 0.006778789684176445, + 0.00039472361095249653, + 0.006761505268514156, + -0.009748183190822601, + -0.022510698065161705, + 0.0007959287613630295, + 0.010591642931103706, + -0.005195573903620243, + -0.04466188699007034, + 0.016523515805602074, + 0.034706294536590576, + -0.0017024751286953688, + -0.021584274247288704, + -0.0024007493630051613, + 0.004932857118546963, + -0.0033409995958209038, + -0.0030056897085160017, + 0.0001562042598379776, + 0.02931368350982666, + 0.002665194682776928, + -0.021210938692092896, + 0.022731931880116463, + -0.02994973585009575, + 0.004417793359607458, + 0.00269112060777843, + -0.010128431022167206, + -0.012154117226600647, + 0.010356579907238483, + -0.004639029037207365, + -0.002022229367867112, + 8.22610963950865e-05, + 0.009907195344567299, + -0.018611423671245575, + -0.008863241411745548, + 0.03531469404697418, + -0.0034844567999243736, + -0.003899272996932268, + -0.000997287454083562, + 0.01880500465631485, + -0.0010638309177011251, + 0.019883526489138603, + -0.020671678707003593, + -0.016122525557875633, + 0.025995153933763504, + 0.01258276030421257, + -0.004193101078271866, + 0.010799050331115723, + 0.024031689390540123, + -0.008268671110272408, + 0.006363973021507263, + 0.005593106150627136, + 0.006961999926716089, + 0.014739805832505226, + 0.018362535163760185, + -0.010349666699767113, + 0.025455892086029053, + -0.005586192477494478, + 0.025317620486021042, + 0.01252745185047388, + -0.004908659495413303, + -0.020975876599550247, + 0.010017813183367252, + 0.007148667238652706, + -0.004452361259609461, + 0.007992126978933811, + -0.004842980299144983, + 0.02842874266207218, + 9.387378668179736e-05, + -0.0012323500122874975, + -0.022773414850234985, + -0.00399260688573122, + 0.02386576309800148, + 0.012644982896745205, + -0.03512111306190491, + -0.017740309238433838, + -0.006312121171504259, + 0.030198626220226288, + -0.00972744170576334, + -0.015527956187725067, + -0.02660355158150196, + 0.006720023695379496, + 0.005309648346155882, + 0.04059668630361557, + -0.013723505660891533, + 0.01696598529815674, + -0.00160482048522681, + -0.025760091841220856, + -0.004690880887210369, + -0.007100271992385387, + -0.010308184660971165, + 0.045712754130363464, + 0.021114148199558258, + 0.04344509169459343, + -0.0020913654007017612, + -0.020077107474207878, + 0.01395165454596281, + -0.037305813282728195, + 0.006274096202105284, + -0.013702765107154846, + -0.031083567067980766, + 0.008254843764007092, + 0.026824787259101868, + 0.004224212374538183, + -0.003622728865593672, + -0.0100592952221632, + 0.027059849351644516, + 0.0326598696410656, + 0.021694891154766083, + -0.007535829208791256, + -0.005828168708831072, + 0.008883981965482235, + -0.017615865916013718, + 0.01609487272799015, + -0.021750200539827347, + 0.004839523229748011, + 0.0184178426861763, + -0.00725928507745266, + 0.034955184906721115, + -0.003615815192461014, + -0.003833593800663948, + -0.0014172890223562717, + -0.0005262981867417693, + 0.01361288782209158, + -0.016772404313087463, + 0.013889431953430176, + 0.03738877549767494, + 0.0004666683089453727, + 0.010404975153505802, + -0.010093863122165203, + -0.010494852438569069, + 0.020284516736865044, + 0.02711515873670578, + -0.023810455575585365, + 0.007929904386401176, + 0.002668651519343257, + 0.028041580691933632, + 0.011193126440048218, + 0.011006458662450314, + -0.004984708968549967, + -0.005510143004357815, + -0.030641095712780952, + -0.0014224741607904434, + 0.023658355697989464, + 0.018196607008576393, + -0.008234103210270405, + 0.01205732673406601, + 0.017173394560813904, + 0.01244448870420456, + -0.00320964097045362, + -0.013585233129560947, + 0.010660778731107712, + 0.020021799951791763, + 0.0047807577066123486, + 0.013011404313147068, + -0.6968913674354553, + -0.01792006380856037, + -0.009679046459496021, + -0.026824787259101868, + 0.01711808517575264, + 0.024861322715878487, + 0.003840507473796606, + -0.0017223517643287778, + -0.01881883293390274, + 0.0005647551151923835, + -0.013813382014632225, + 0.023395638912916183, + 0.0054997727274894714, + 0.016164008527994156, + -0.01711808517575264, + -0.009623738005757332, + 0.00045500160194933414, + -0.01671709679067135, + -0.02618873491883278, + 0.03539765626192093, + -0.010273616760969162, + 0.02735022082924843, + 0.013605974614620209, + 0.011967449449002743, + 0.019648464396595955, + -0.004670139867812395, + 0.010162998922169209, + -0.034042589366436005, + -0.018085990101099014, + -0.013370911590754986, + -0.020837604999542236, + 0.006253355648368597, + 0.026299351826310158, + -0.0063570598140358925, + 0.06294145435094833, + 0.007065704092383385, + -0.03426382690668106, + 0.013668196275830269, + 0.01688302308320999, + 0.0295625738799572, + -0.01724253036081791, + -0.01128300279378891, + 0.015444993041455746, + -0.019952664151787758, + -0.005402982234954834, + 0.009430157020688057, + 0.025400584563612938, + 0.004224212374538183, + -0.00036253215512260795, + -0.015348202548921108, + 0.00033747032284736633, + -0.009907195344567299, + -0.005707180593162775, + 0.021971436217427254, + -0.014242026023566723, + 0.0006027799099683762, + 0.02864997833967209, + -0.008655833080410957, + 0.003169887699186802, + 0.029092449694871902, + 0.002946923952549696, + 0.009831146337091923, + -0.007266198750585318, + -0.003318530274555087, + 0.0057970574125647545, + 0.0004476559115573764, + -0.023395638912916183, + 0.024806013330817223, + 0.010667691938579082, + -0.0031612457241863012, + 0.020726986229419708, + -0.0005716686719097197, + -0.03216208890080452, + -0.0030765540432184935, + 0.02959022857248783, + 0.017698828130960464, + 0.01905389502644539, + -0.03147072717547417, + -0.008123485371470451, + 0.009139785543084145, + 0.01205732673406601, + 0.005897304974496365, + -0.017933890223503113, + 0.006412368267774582, + 0.02270427718758583, + 0.012492883950471878, + -0.03556358441710472, + 0.0147812869399786, + 0.0026116142980754375, + -0.003771371440961957, + 0.017989199608564377, + 0.01973142847418785, + -0.0013204985298216343, + -0.010031640529632568, + -0.0004167607403360307, + 0.004518040921539068, + -0.02476453222334385, + 0.02346477471292019, + 0.01921982131898403, + -0.03061344288289547, + -0.004839523229748011, + 0.009450897574424744, + 0.0052923643961548805, + -0.0060632312670350075, + -0.009782751090824604, + 0.02087908610701561, + -0.0009385218727402389, + 0.009271143935620785, + 0.032245051115751266, + -0.03360011801123619, + -0.017353147268295288, + 0.005347673315554857, + -0.012810910120606422, + -0.022386252880096436, + 0.0011640782468020916, + -0.03036455251276493, + 0.01187757309526205, + -0.0004219459369778633, + 0.0051852036267519, + -0.013695850968360901, + 0.018832659348845482, + -0.02401786297559738, + 0.013308688998222351, + -0.0009791392367333174, + 0.02140451967716217, + 3.362280494911829e-06, + 0.0026842071674764156, + -0.011255349032580853, + -0.018735868856310844, + -0.011849918402731419, + -0.0027118613943457603, + 0.0060563175939023495, + 0.02231711708009243, + -0.006080515217036009, + 0.009492379613220692, + 0.0020930939354002476, + 0.01460153330117464, + 0.004438534379005432, + 0.010729914531111717, + -0.008047436363995075, + -0.040430761873722076, + -0.007681014947593212, + -0.0016221045516431332, + -0.015652401372790337, + -0.020657850429415703, + -0.015182276256382465, + -0.02826281636953354, + -0.0009480280568823218, + 0.005600019823759794, + 0.014878077432513237, + -0.004542238544672728, + 0.004200014751404524, + -0.017864754423499107, + 0.018238089978694916, + 0.009706701152026653, + -0.01957932859659195, + -0.018321052193641663, + -0.027211949229240417, + -0.021999090909957886, + -0.002004945417866111, + 0.005983724724501371, + 0.019524019211530685, + -0.02761293761432171, + 0.01696598529815674, + 0.012223253026604652, + -0.005945700220763683, + 0.007356075569987297, + 0.004286434967070818, + -0.006433109287172556, + -0.032770488411188126, + 0.012755600735545158, + -0.01595659926533699, + -0.005323475692421198, + 0.002297045197337866, + -0.0164820346981287, + 0.008047436363995075, + -0.017809445038437843, + 0.008794105611741543, + -0.023395638912916183, + -0.008932377211749554, + 0.01400004979223013, + 0.007784719113260508, + -0.015873637050390244, + -0.0071279266849160194, + 0.01594277285039425, + 0.004217298701405525, + -0.00693780230358243, + 0.022925512865185738, + 0.001214201794937253, + 0.002212353516370058, + 0.0026012437883764505, + -0.006464220583438873, + -0.004390139132738113, + 0.011110163293778896, + -0.021045012399554253, + -0.012555106543004513, + 0.005862737074494362, + 0.009450897574424744, + -0.010736828669905663, + 0.027792690321803093, + 0.024086998775601387, + 0.01400004979223013, + 0.021045012399554253, + -0.033683083951473236, + 0.025345275178551674, + -0.024460334330797195, + 0.0033755674958229065, + -0.03332357481122017, + 0.022220326587557793, + 0.021459829062223434, + 0.008261757902801037, + -0.019385747611522675, + -0.005530884023755789, + 0.004096311051398516, + 0.00122457230463624, + 0.017878582701086998, + 0.015348202548921108, + 0.023810455575585365, + 0.01620548963546753, + -0.0017612407682463527, + -0.0027913679368793964, + 0.011725474148988724, + -0.006623233202844858, + 0.01736697554588318, + -0.0010474111186340451, + -0.0033012463245540857, + 0.006910148076713085, + 0.046653006225824356, + 0.009056822396814823, + -0.015444993041455746, + -0.0024698853958398104, + 0.007763978093862534, + -0.005461747758090496, + 0.016647960990667343, + -0.005582735873758793, + -0.01019065361469984, + 0.010349666699767113, + -0.01037732046097517, + 0.023188229650259018, + 0.0016151908785104752, + 0.00013935234164819121, + -0.00036404450656846166, + 0.04471719637513161, + -0.004822239279747009, + 0.005762489512562752, + 0.002153587993234396, + 0.011849918402731419, + -0.021874645724892616, + -0.005105697084218264, + -0.005043474491685629, + -0.029507264494895935, + 0.006180762778967619, + -0.028179852291941643, + 0.019938835874199867, + 0.02477835863828659, + -0.004853350576013327, + 0.005821255035698414, + 0.009457811713218689, + 0.007362989243119955, + 0.027571456506848335, + 0.02190230041742325, + 0.0020567975006997585, + 0.011725474148988724, + -0.0022400079760700464, + -0.014172890223562717, + -0.0022175386548042297, + -0.0011243249755352736, + -0.005223228596150875, + -0.012161030434072018, + -0.008773364126682281, + -0.009679046459496021, + 0.020063281059265137, + 0.0122647350654006, + 0.0042034718208014965, + 0.01660647802054882, + -0.006284466944634914, + 0.002872602781280875, + 0.011435102671384811, + 0.004877548199146986, + -0.004217298701405525, + -0.011690905317664146, + -0.03232801705598831, + 0.005748662166297436, + -0.008932377211749554, + -0.003465444315224886, + 0.0009039537981152534, + -0.04355571046471596, + -0.003142233472317457, + 0.017947718501091003, + -0.0045145838521420956, + 0.0052266851998865604, + 0.023119093850255013, + -0.0015235856408253312, + -0.010653864592313766, + -0.002169143408536911, + 0.012555106543004513, + 0.003840507473796606, + -0.01697981357574463, + -0.01945488341152668, + -0.0055792792700231075, + -0.0019133401801809669, + -0.019026240333914757, + -0.013149675913155079, + 0.008870154619216919, + 0.03185788914561272, + -0.011787695810198784, + 0.0029088992159813643, + -0.023450948297977448, + 0.0118015231564641, + 0.0021916127298027277, + 0.009132872335612774, + 0.009513120166957378, + -0.013087454251945019, + -0.0055723655968904495, + 0.014974867925047874, + -0.0063224914483726025, + -0.010667691938579082, + -0.003581247292459011, + 0.023146748542785645, + 0.01671709679067135, + -0.0050469315610826015, + -0.009775836952030659, + -0.008600524626672268, + 0.009174353443086147, + 0.039628781378269196, + 0.015873637050390244, + 0.006398541387170553, + 0.02696305885910988, + -0.005458291154354811, + 0.012340785004198551, + -0.0342085175216198, + -0.014096840284764767, + 0.01751907542347908, + -0.0016730923671275377, + -0.011054853908717632, + -0.022925512865185738, + 0.00967213325202465, + -0.016039563342928886, + 0.03926927596330643, + -0.009582255966961384, + 0.01122769434005022, + -0.01348844263702631, + 0.03440209850668907, + -0.016039563342928886, + -0.004652855917811394, + 0.0050469315610826015, + 0.010197566822171211, + -0.00028605040279217064, + 0.011780782602727413, + -0.006281009875237942, + -0.007646447047591209, + 0.03257690742611885, + 0.0022797612473368645, + -0.03700161352753639, + 0.021363038569688797, + 0.010183739475905895, + -0.0024906261824071407, + 0.013626715168356895, + -0.0010422258637845516, + 0.013052885420620441, + 0.001489881775341928, + -0.011013372801244259, + 0.03335122764110565, + 0.0037540874909609556, + 0.01753290183842182, + 0.011849918402731419, + 0.0005893848137930036, + -0.0068548391573131084, + 0.016800059005618095, + 0.007404470816254616, + -0.00362964253872633, + 0.010750655084848404, + 0.01597042754292488, + 0.004099767655134201, + -0.009782751090824604, + 0.010653864592313766, + 0.008586697280406952, + -0.006879036780446768, + -0.0031301346607506275, + 0.015154621563851833, + 0.004407423082739115, + -0.011103249154984951, + 0.012416834011673927, + -0.020160071551799774, + -0.014145235531032085, + -0.023409465327858925, + 0.021985262632369995, + -0.01659265160560608, + -0.003029887331649661, + -0.029368992894887924, + -0.011006458662450314, + -0.013052885420620441, + -0.010031640529632568, + -0.0020861802622675896, + 0.0013576591154560447, + 0.0002335502067580819, + -0.008206448517739773, + 0.0054997727274894714, + 0.013363998383283615, + -0.007729410193860531, + 0.008579783141613007, + 0.0007911756401881576, + 0.016634132713079453, + 0.007812373340129852, + -0.008662747219204903, + -0.02320205792784691, + -0.00876645091921091, + -0.0055723655968904495, + -0.0016290181083604693, + 0.03619963303208351, + 0.02271810546517372, + -0.007224717177450657, + -0.007549656555056572, + -0.010660778731107712, + 0.013799555599689484, + -0.0005098783294670284, + 0.006305207498371601, + -0.017961544916033745, + 0.0053200190886855125, + -0.03100060299038887, + 0.01829339750111103, + 0.01097189076244831, + -0.010716087184846401, + 0.0006278417422436178, + -0.006996567826718092, + -0.023934898898005486, + 0.003121492452919483, + -0.013004490174353123, + 0.024045517668128014, + 0.00583508238196373, + 0.025621820241212845, + 0.011628683656454086, + -0.026244044303894043, + -0.021694891154766083, + 0.028567014262080193, + -0.0052197715267539024, + 0.028221335262060165, + 0.002810380421578884, + 0.008476079441606998, + 0.002563219051808119, + 0.002997047733515501, + 0.03998829051852226, + -0.012658810243010521, + -0.040154214948415756, + -0.01608104445040226, + -0.014200543984770775, + 0.015541783533990383, + 0.022054398432374, + 0.008662747219204903, + 0.0011104977456852794, + 0.00492940004914999, + -0.02553885616362095, + -0.007093358319252729, + -0.019676119089126587, + -0.011234607547521591, + 0.006820271257311106, + -0.00894620455801487, + -0.02346477471292019, + -0.020851431414484978, + -0.023506255820393562, + -0.02180550992488861, + 0.0045145838521420956, + -0.0028570471331477165, + -0.006844468880444765, + -0.024999594315886497, + -0.003062726929783821, + 0.005368414334952831, + -0.02322971262037754, + -0.033157650381326675, + -0.029092449694871902, + -0.01031509879976511, + 0.008455338887870312, + 0.0021967978682368994, + 0.005427179858088493, + -0.010660778731107712, + 0.029258375987410545, + -0.00412396527826786, + 0.013343256898224354, + 0.0013386467471718788, + -0.000843459798488766, + 0.0017258086008951068, + -0.015223757363855839, + 0.021349212154746056, + 0.024736877530813217, + 0.023782800883054733, + 0.022344769909977913, + 0.01646820642054081, + -0.017159566283226013, + -0.011324484832584858, + -0.0147812869399786, + -0.014062272384762764, + -0.01200893148779869, + -0.024819841608405113, + 0.025732437148690224, + 0.020533407106995583, + 0.020671678707003593, + -0.0010128431022167206, + 0.004607917740941048, + -0.0013896345626562834, + 0.021833164617419243, + -0.02840108796954155, + -0.006298293825238943, + -0.008448424749076366, + -0.0032009989954531193, + 0.011117076501250267, + -0.011594114825129509, + -0.020671678707003593, + 0.013087454251945019, + 0.011193126440048218, + 0.0050469315610826015, + 0.007100271992385387, + -0.002080995123833418, + -0.00019142043311148882, + -0.008393116295337677, + 0.023768972605466843, + 0.003778285114094615, + 0.023893417790532112, + 0.003989149816334248, + 0.004137792624533176, + -0.007404470816254616, + -0.023699836805462837, + 0.013142762705683708, + -0.022745760157704353, + 0.010287444107234478, + 0.011345225386321545, + 0.006944715976715088, + -0.006903234403580427, + -0.026548242196440697, + -0.00207408145070076, + 0.0371398851275444, + 0.001424202579073608, + -0.011732387356460094, + 0.020920567214488983, + -0.02439119853079319, + -0.018583768978714943, + -0.030530478805303574, + -0.012382266111671925, + -0.028083061799407005, + -0.001611734158359468, + 0.005295821465551853, + -0.01790623553097248, + 0.0207408145070076, + -0.0060770586133003235, + -0.018611423671245575, + 0.01568005606532097, + -0.00817188061773777, + 0.019551673904061317, + -0.010854359716176987, + 0.03205147013068199, + 0.01818278059363365, + 0.009381761774420738, + -0.012181771919131279, + 0.02346477471292019, + 0.013605974614620209, + 0.006643974222242832, + 0.013087454251945019, + 0.020284516736865044, + 0.002274575876072049, + -0.014435607008635998, + 0.0103704072535038, + 0.02271810546517372, + -0.014110667631030083, + -0.04543621093034744, + 0.026230216026306152, + 0.02606428973376751, + 0.011041026562452316, + -0.018085990101099014, + 0.007376816123723984, + -0.03725050389766693, + 0.02826281636953354, + 0.016412897035479546, + -0.01019065361469984, + -0.011725474148988724, + -0.012548192404210567, + -0.04903128370642662, + 0.00999015849083662, + -0.037278156727552414, + 0.011587201617658138, + 0.002566675888374448, + -0.027211949229240417, + -0.014324989169836044, + -0.021432174369692802, + -0.013474615290760994, + 0.004490386229008436, + -0.007062247488647699, + 0.022980822250247, + 0.015334375202655792, + 0.0118015231564641, + 0.003029887331649661, + -0.004227669443935156, + -0.032383326441049576, + -0.014615360647439957, + 0.015348202548921108, + 0.022870205342769623, + -0.010211394168436527, + -0.024709222838282585, + -0.0007246321765705943, + 0.002881244756281376, + 0.006453849840909243, + -0.02555268257856369, + -0.013343256898224354, + -0.008220275864005089, + -0.018390187993645668, + -0.0064019979909062386, + -0.007674101274460554, + 0.015569438226521015, + -0.03763766586780548, + -0.01023904886096716, + -0.012568933889269829, + -0.0012358068488538265, + -0.027308739721775055, + -0.004894832149147987, + 0.021363038569688797, + 0.009188180789351463, + -0.01439412496984005, + 0.0005876563955098391, + 0.013903259299695492, + 0.009526947513222694, + 0.014159062877297401, + -0.027046021074056625, + 0.013723505660891533, + 0.01543116569519043, + -0.027972444891929626, + 0.01635758951306343, + -0.008704228326678276, + 0.014919559471309185, + -0.008268671110272408, + -0.0037264330312609673, + -0.010522506199777126, + -0.02400403656065464, + 0.0028242075350135565, + -0.02631318010389805, + -0.005644958466291428, + -0.021224766969680786, + 0.002412848174571991, + 0.02553885616362095, + 0.009423242881894112, + -0.008849414065480232, + 0.00796447228640318, + 0.0030488998163491488, + 0.00451112724840641, + -0.03904803842306137, + 0.0034153207670897245, + 0.004300262313336134, + -0.026299351826310158, + 0.005845452658832073, + -0.0037886553909629583, + -0.013011404313147068, + 0.004614831414073706, + 0.003581247292459011, + 0.007515088189393282, + 0.015417338348925114, + -0.03036455251276493, + 0.008317066356539726, + 0.00021885879687033594, + 0.020547233521938324, + -0.02632700651884079, + -0.023962553590536118, + -0.02569095604121685, + 0.00017024751286953688, + -0.030060352757573128, + 0.014808941632509232, + -0.018528461456298828, + -0.004542238544672728, + -0.0034809999633580446, + 0.01433881651610136, + -0.011974363587796688, + 0.023008476942777634, + -0.004950141068547964, + -0.015292894095182419, + 0.0015806228620931506, + -0.008489906787872314, + -0.014242026023566723, + -0.01525141205638647, + -0.011144731193780899, + 0.004628658294677734, + 0.014311161823570728, + -0.015154621563851833, + -0.008158053271472454, + 0.009762009605765343, + -0.01880500465631485, + 0.0038543345872312784, + -0.01830722577869892, + 0.010812877677381039, + 0.014836596325039864, + -0.019938835874199867, + -0.020325997844338417, + 0.023395638912916183, + -0.004276064690202475, + 0.020962050184607506, + -0.022662796080112457, + -0.020837604999542236, + 0.016675613820552826, + 0.015735363587737083, + 0.014643015339970589, + -0.01413140818476677, + -0.017449937760829926, + -0.030060352757573128, + 0.026617377996444702, + 0.009236576035618782, + 0.020796123892068863, + 0.01635758951306343, + -0.014629187993705273, + -0.0063224914483726025, + -0.013267207890748978, + -0.004151619505137205, + 0.018072163686156273, + -0.011345225386321545, + 0.0031595174223184586, + -0.0068410118110477924, + 0.0023644529283046722, + 0.005530884023755789, + 0.007314593996852636, + -0.01343313418328762, + 0.014255853369832039, + -0.006837555207312107, + -0.010045467875897884, + -0.00596644077450037, + 0.021556619554758072, + -0.003688408061861992, + 0.006903234403580427, + -0.020146245136857033, + 0.027198120951652527, + -0.014836596325039864, + -0.013142762705683708, + 0.030005045235157013, + 0.009042995050549507, + -0.01674475148320198, + -0.02179168164730072, + 0.0005833354080095887, + -0.016025735065340996, + -0.008268671110272408, + -0.00111913972068578, + -0.027917135506868362, + -0.0017119813710451126, + -0.015666227787733078, + 0.020809950307011604, + -0.004504213575273752, + 0.012631155550479889, + -0.013758073560893536, + -0.0236030463129282, + -0.0002549391647335142, + 0.021376866847276688, + 0.010494852438569069, + -0.01880500465631485, + -0.00561384717002511, + 0.012375352904200554, + 0.008752623572945595, + -0.0029953191988170147, + -0.009374847635626793, + -0.00876645091921091, + -0.0066024926491081715, + 0.004531867802143097, + 0.018763523548841476, + -0.01356449257582426, + -0.013453874737024307, + 0.01153880637139082, + 0.02645145170390606, + -0.008579783141613007, + 0.012762514874339104, + 0.1973419338464737, + -0.0035432225558906794, + 0.00051031046314165, + 0.022469215095043182, + -0.010563988238573074, + -0.00466668326407671, + 0.022510698065161705, + -0.0006325948634184897, + -0.011041026562452316, + 0.00992793682962656, + -0.006118540186434984, + 0.017864754423499107, + -0.008234103210270405, + 0.002843220019713044, + -0.012935354374349117, + -0.00798521377146244, + -0.033129993826150894, + -0.006592122372239828, + -0.029507264494895935, + -0.029645537957549095, + 0.014324989169836044, + -0.009499292820692062, + 0.0004947548150084913, + 0.007888423278927803, + 0.03127714619040489, + -8.506974700139835e-05, + -0.00725928507745266, + -0.010674606077373028, + 0.027709728106856346, + 0.027599111199378967, + 0.0018822288839146495, + -0.02244156040251255, + -0.008842500858008862, + 0.0015538326697424054, + -0.024930458515882492, + 0.012230167165398598, + -0.008669660426676273, + -0.013080540113151073, + 0.021100321784615517, + -0.0019358093850314617, + 0.002500996459275484, + -0.00014756225573364645, + 0.003292604349553585, + -0.006910148076713085, + -0.011531893163919449, + 0.0061323675327003, + 0.00651952950283885, + -0.00563804479315877, + 0.011013372801244259, + 0.017588211223483086, + -0.0345127135515213, + -0.0008577191038057208, + 0.014449434354901314, + 0.014684496447443962, + 0.0047150785103440285, + 0.0011312385322526097, + 0.013287948444485664, + 0.015735363587737083, + -0.004231126047670841, + -0.004002977162599564, + 0.005178289953619242, + 0.04466188699007034, + -0.0077501507475972176, + 0.02945195697247982, + -0.029147757217288017, + -0.01594277285039425, + 0.005610390100628138, + 0.012485969811677933, + 0.00399260688573122, + -0.005682982970029116, + -0.012361525557935238, + -0.012810910120606422, + 0.0015771661419421434, + 0.0004260508867446333, + -0.015292894095182419, + -0.03304703161120415, + 0.016412897035479546, + 0.020699333399534225, + -0.00829632580280304, + 0.017021294683218002, + -0.01245140191167593, + -0.014587705954909325, + -0.0009523490443825722, + 0.009713614359498024, + -0.006035577040165663, + -0.032217398285865784, + 0.02129390276968479, + 0.03697395697236061, + -0.0013101281365379691, + -0.028871214017271996, + -0.007006938569247723, + -0.00037614331813529134, + -0.011939795687794685, + -0.0017024751286953688, + -0.008842500858008862, + 0.01818278059363365, + -0.033683083951473236, + 0.014836596325039864, + 0.005181746557354927, + 0.013149675913155079, + -0.030834676697850227, + -0.01968994550406933, + 0.017463766038417816, + 0.006443479564040899, + -0.019510192796587944, + -0.006664715241640806, + 0.0021967978682368994, + 0.016841541975736618, + -0.0013974123867228627, + -0.008026694878935814, + 0.002001488581299782, + -0.009326453320682049, + -0.0032372954301536083, + -0.007694842293858528, + 0.028193680569529533, + -0.02465391531586647, + -0.023644529283046722, + 0.008489906787872314, + 0.011815350502729416, + -0.008420770987868309, + 0.017284011468291283, + -0.002025686204433441, + 0.007840028032660484, + 0.0058247121050953865, + -0.012548192404210567, + -0.014366471208631992, + -0.025193175300955772, + -0.011766955256462097, + 0.006792616564780474, + -0.0394628569483757, + 9.700651571620256e-05, + -0.012686464935541153, + 0.01634376123547554, + -0.00920200813561678, + -0.016938332468271255, + 0.003640012815594673, + 0.005561994854360819, + -0.017062775790691376, + 0.0009609910775907338, + -0.0038681619334965944, + 0.002497539622709155, + 0.014255853369832039, + -0.006215330678969622, + 0.014850422739982605, + 0.005945700220763683, + -0.009443984366953373, + 0.03545296564698219, + -0.003643469652161002, + -0.009001513011753559, + -0.014684496447443962, + -0.027543801814317703, + -0.0005937058012932539, + -0.020782295614480972, + -0.020823776721954346, + 0.023063786327838898, + -0.017463766038417816, + -0.009381761774420738, + -0.039905328303575516, + 0.010951150208711624, + -0.009195093996822834, + -0.019938835874199867, + -0.012631155550479889, + 0.022220326587557793, + -0.008697315119206905, + -0.03216208890080452, + -0.01245140191167593, + -0.18229793012142181, + 0.01984204538166523, + 0.008282498456537724, + -0.009188180789351463, + 0.03061344288289547, + -0.0003376863896846771, + 0.02152896486222744, + 0.00992793682962656, + -0.0050400178879499435, + 0.015057831071317196, + -0.009685960598289967, + -0.02349242940545082, + -0.03216208890080452, + -0.018321052193641663, + -0.003133591264486313, + 0.008144226856529713, + 0.01688302308320999, + -0.009049908258020878, + 0.030945295467972755, + 0.011041026562452316, + 0.026423797011375427, + -0.009782751090824604, + 0.026368489488959312, + -0.001089756959117949, + 0.014878077432513237, + 0.019358092918992043, + -0.008579783141613007, + 0.030558133497834206, + 0.0021950695663690567, + -0.0024664285592734814, + -0.012824736535549164, + -0.0013308689231052995, + 0.01775413751602173, + -0.007667187601327896, + -0.0029434673488140106, + -0.009250403381884098, + 0.02216501720249653, + -0.017809445038437843, + 0.0010119789512827992, + 0.0066681718453764915, + 0.0066819991916418076, + 0.035342346876859665, + -0.0014267951482906938, + -0.010557074099779129, + 0.016025735065340996, + 0.004625201690942049, + 0.04037545248866081, + 0.01739463023841381, + 0.004234583117067814, + 0.0031975421588867903, + -0.011117076501250267, + -0.023948727175593376, + 0.012672637589275837, + 0.00039774831384420395, + -0.011891400441527367, + -0.008337806910276413, + -0.004403966479003429, + 0.003657296998426318, + 0.0004791992250829935, + 0.003107665339484811, + -0.001880500465631485, + -0.031609002500772476, + 0.007369902916252613, + -0.01660647802054882, + -0.0003955878200940788, + -0.01608104445040226, + -0.031581345945596695, + 0.021335383877158165, + -0.007245457731187344, + -0.003325443947687745, + 0.004300262313336134, + -0.0011632139794528484, + 0.033268265426158905, + -0.022372424602508545, + 0.003396308282390237, + -0.004255323670804501, + -0.029230721294879913, + 0.00015944501501508057, + 0.009423242881894112, + -0.014013877138495445, + 0.007950645871460438, + 0.04278138652443886, + -0.011241521686315536, + -0.007418298162519932, + 0.002884701592847705, + 0.02646527998149395, + 0.0007224716828204691, + -0.0038681619334965944, + -0.003373839193955064, + -0.020201552659273148, + 0.01309436745941639, + -0.00972052849829197, + -0.015320547856390476, + -0.03075171448290348, + 0.004607917740941048, + 0.009879541583359241, + -0.005510143004357815, + 0.011013372801244259, + -0.0010785222984850407, + 0.00022188349976204336, + 0.005001992918550968, + 0.003328900784254074, + -0.04167520999908447, + 0.015998082235455513, + 0.03957347199320793, + 0.013675110414624214, + -0.004037545062601566, + 0.021888472139835358, + 0.026935404166579247, + 0.00954077485948801, + -0.03138776496052742, + 0.005396068561822176, + 0.02959022857248783, + 0.009236576035618782, + -0.003347913036122918, + 0.008220275864005089, + 0.01205732673406601, + -0.012292389757931232, + 0.0030108748469501734, + 0.007715582847595215, + 0.03512111306190491, + -0.006139281205832958, + -0.007902250625193119, + -0.001807907596230507, + -0.0009990158723667264, + 0.0005962984287180007, + -0.07499878108501434, + -0.016634132713079453, + 0.0015935859410092235, + 0.025649473071098328, + -0.0009082748438231647, + -0.005534340627491474, + 0.0010621024994179606, + 0.03387666493654251, + -0.03440209850668907, + 0.03166430816054344, + -0.01725635677576065, + -0.033295921981334686, + 0.004645942244678736, + -0.024612432345747948, + 0.03346184641122818, + 0.02606428973376751, + -0.001142473192885518, + -0.0009367934544570744, + -0.019413402304053307, + 0.025372929871082306, + -0.0031232209876179695, + -0.006851382553577423, + -0.007922991178929806, + 0.0012150660622864962, + -0.004397052805870771, + 0.007611878681927919, + -0.018113644793629646, + 0.02840108796954155, + 0.005873107351362705, + -0.0017785248346626759, + -0.005717551335692406, + -0.00790916383266449, + 0.029783809557557106, + -0.024930458515882492, + 0.00473236246034503, + 0.003767914604395628, + -0.00401680450886488, + -0.013073626905679703, + 0.02462626062333584, + -0.02219267189502716, + -0.0053131054155528545, + 0.01960698328912258, + -0.010024727322161198, + -0.009084477089345455, + 0.010232134722173214, + 0.008088917471468449, + -0.006312121171504259, + 0.015458820387721062, + 0.010550160892307758, + -0.01905389502644539, + -0.051962655037641525, + -0.016758577898144722, + -0.023962553590536118, + -0.018390187993645668, + 0.05304117500782013, + -0.02440502494573593, + 0.0033721106592565775, + 0.015541783533990383, + -0.011186212301254272, + 0.014352643862366676, + -0.007356075569987297, + -0.011960536241531372, + -0.012333870865404606, + 0.03501049429178238, + 0.026506761088967323, + 0.009699787944555283, + -0.009029167704284191, + 0.00014713015116285533, + 0.027419356629252434, + -0.009319539181888103, + -0.004099767655134201, + 0.010301271453499794, + -0.03202381730079651, + 0.03254925087094307, + -0.005378784611821175, + -0.005434093531221151, + -0.010619296692311764, + -0.010681519284844398, + 0.0009367934544570744, + -0.027142811566591263, + -0.0001493986783316359, + -0.027391701936721802, + -0.01982821896672249, + -0.005454834084957838, + 0.010543247684836388, + 0.014477088116109371, + 0.026935404166579247, + 0.02242773398756981, + 0.01931661181151867, + -0.032383326441049576, + 0.0016817343421280384, + 0.02800009958446026, + 0.008344721049070358, + -0.024944286793470383, + -0.0009730898891575634, + -2.2104630261310376e-05, + 0.015223757363855839, + -0.008317066356539726, + -0.002329884795472026, + 0.012257820926606655, + -0.02840108796954155, + -0.027599111199378967, + -0.04568510130047798, + 0.02867763303220272, + -0.0033686538226902485, + -0.006529899779707193, + -0.002606428926810622, + -0.0100592952221632, + 0.016440551728010178, + -0.005620760843157768, + -0.010563988238573074, + 0.004321002867072821, + -0.03487222269177437, + 0.006979283876717091, + -0.018445497378706932, + -0.020823776721954346, + -0.034568022936582565, + -0.014712151139974594, + 0.025317620486021042, + 0.0021674151066690683, + 0.022206498309969902, + 0.0011511151678860188, + 0.006139281205832958, + -6.87039500917308e-05, + 0.02086525969207287, + -0.030170971527695656, + -0.02826281636953354, + -0.002259020460769534, + -0.02347860112786293, + 0.015610919333994389, + -0.025234658271074295, + -0.025220829993486404, + 0.025649473071098328, + -0.028373433277010918, + -0.016039563342928886, + 0.007197062484920025, + 0.0031595174223184586, + -0.01109633594751358, + 0.0013965481193736196, + -0.0035397657193243504, + 0.024197617545723915, + 0.004002977162599564, + -0.015984253957867622, + -0.03426382690668106, + 0.011704732663929462, + -0.010169913060963154, + 0.0003726865106727928, + 0.01011460367590189, + -0.013585233129560947, + -0.016302280128002167, + 0.04109446704387665, + -0.0007030271808616817, + 0.010349666699767113, + 0.020395133644342422, + -0.0035328520461916924, + -0.002770627150312066, + -5.908971434109844e-05, + -0.028622323647141457, + 0.012734860181808472, + 0.00011450657621026039, + -0.011338312178850174, + -0.029258375987410545, + 0.0037471738178282976, + 0.019925009459257126, + 0.01291461382061243, + 0.003657296998426318, + -0.016952158883213997, + 0.006301750894635916, + -0.013122022151947021, + -0.004552608821541071, + 0.019772909581661224, + -0.021556619554758072, + -0.029147757217288017, + 0.018749695271253586, + -0.008289412595331669, + 0.0014587705954909325, + 0.009907195344567299, + -0.005717551335692406, + -0.012672637589275837, + 0.004286434967070818, + -0.011552633717656136, + 0.02451564185321331, + 0.027543801814317703, + -0.010653864592313766, + -0.005956070497632027, + 0.012762514874339104, + 0.02049192413687706, + 0.014974867925047874, + 0.005520513281226158, + -0.0147812869399786, + 0.0011606214102357626, + 0.015915118157863617, + -0.025262311100959778, + 0.003913100343197584, + -0.012824736535549164, + 0.041177429258823395, + 0.002635811921209097, + 0.005475575104355812, + -0.011753127910196781, + -0.002661737846210599, + 0.026534415781497955, + -4.9826565373223275e-05, + 0.00725928507745266, + -0.010695346631109715, + 0.009160526096820831, + -0.003771371440961957, + -0.0069343456998467445, + -0.022676624357700348, + -0.015527956187725067, + -0.03138776496052742, + -0.007535829208791256, + 0.00876645091921091, + 0.016509687528014183, + -0.010681519284844398, + -0.0068410118110477924, + 0.018500806763768196, + 0.007079531438648701, + 0.00021118902077432722, + -0.00011364238162059337, + 0.002314329147338867, + -0.01608104445040226, + 0.017228703945875168, + 0.019800564274191856, + 0.023188229650259018, + 0.027654418721795082, + -0.007681014947593212, + -0.006733851041644812, + 0.004037545062601566, + 0.0003646926488727331, + -0.004092853982001543, + -0.0047219921834766865, + -0.01711808517575264, + 0.006177305709570646, + -0.022137362509965897, + -0.020989704877138138, + -0.026672687381505966, + -0.020049454644322395, + 0.006121997255831957, + 0.02336798422038555, + 0.010328925214707851, + -0.006529899779707193, + 0.06664714962244034, + 0.013841036707162857, + -0.014961040578782558, + 0.0028639608062803745, + 0.006277553271502256, + 0.0193442665040493, + 0.017975373193621635, + 0.005371870938688517, + -0.006737307645380497, + -0.025207003578543663, + 0.03459567949175835, + 0.004227669443935156, + -0.004632115364074707, + -0.00804052222520113, + 0.0036123585887253284, + -0.00554816797375679, + -0.01646820642054081, + 0.01751907542347908, + -0.017685001716017723, + -0.004317546263337135, + 0.0049605113454163074, + 0.00946472492069006, + 0.015610919333994389, + -0.013149675913155079, + -0.011372880078852177, + 0.0014812398003414273, + -0.00023808726109564304, + -0.0037851985543966293, + -0.005281994119286537, + -0.047952763736248016, + -0.0045076701790094376, + -0.0012591403210535645, + -0.017159566283226013, + -0.02140451967716217, + 0.02864997833967209, + -0.015002522617578506, + -0.014297334477305412, + 0.008441511541604996, + 0.00444544805213809, + 0.022648969665169716, + -0.017740309238433838, + 0.011663251556456089, + -0.01971760019659996, + -0.020339826121926308, + 0.003847421146929264, + 0.0044419909827411175, + -0.01685536839067936, + 0.0053131054155528545, + -0.020906740799546242 + ] + } + ], + "model": "text-embedding-ada-002-v2", + "usage": { + "prompt_tokens": 2, + "total_tokens": 2 + } +} diff --git a/chirps/embedding/forms.py b/chirps/embedding/forms.py new file mode 100644 index 00000000..4a4e53c2 --- /dev/null +++ b/chirps/embedding/forms.py @@ -0,0 +1,22 @@ +"""Forms for the embedding application models.""" +from django import forms + +from .models import Embedding + + +class CreateEmbeddingForm(forms.Form): + """Form for creating embeddings.""" + + text = forms.CharField( + label='Text', + max_length=1000, + required=True, + widget=forms.TextInput(attrs={'class': 'form-control', 'placeholder': 'The text to embed'}), + ) + model = forms.CharField( + label='Model', + max_length=100, + required=True, + widget=forms.TextInput(attrs={'class': 'form-control', 'placeholder': 'The model to embed the text with'}), + ) + service = forms.ChoiceField(label='Service', required=True, choices=Embedding.Service.choices) diff --git a/chirps/embedding/migrations/0001_initial.py b/chirps/embedding/migrations/0001_initial.py new file mode 100644 index 00000000..816c6548 --- /dev/null +++ b/chirps/embedding/migrations/0001_initial.py @@ -0,0 +1,51 @@ +# Generated by Django 4.2.2 on 2023-07-11 17:27 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Embedding', + fields=[ + ( + 'id', + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID', + ), + ), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('model', models.CharField(max_length=256)), + ( + 'service', + models.CharField( + choices=[ + ('OA', 'OpenAI'), + ('LH', 'Locally Hosted: NOT IMPLEMENTED'), + ], + max_length=2, + ), + ), + ('text', models.TextField()), + ('vectors', models.JSONField()), + ( + 'user', + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + ), + ] diff --git a/chirps/embedding/migrations/__init__.py b/chirps/embedding/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/chirps/embedding/models.py b/chirps/embedding/models.py new file mode 100644 index 00000000..7fdeba81 --- /dev/null +++ b/chirps/embedding/models.py @@ -0,0 +1,52 @@ +"""Models for the embedding app""" +from django.contrib.auth.models import User +from django.db import models + +from .providers.base import BaseEmbeddingProvider +from .providers.openai import OpenAIEmbeddingProvider + + +class Embedding(models.Model): + """A model to store embeddings generated by a target.""" + + class Service(models.TextChoices): + """Enumerations to define services available for generating embeddings.""" + + OPEN_AI = 'OA', 'OpenAI' + LOCAL = 'LH', 'Locally Hosted: NOT IMPLEMENTED' + + @classmethod + def get_provider_from_service_name(cls, name: str) -> BaseEmbeddingProvider: + """Get the provider for the service.""" + if name == Embedding.Service.OPEN_AI: + return OpenAIEmbeddingProvider() + + raise NotImplementedError + + created_at = models.DateTimeField(auto_now_add=True) + + # Name of the model used to generate the embedding + model = models.CharField(max_length=256) + + # The service used to generate the embedding + service = models.CharField(max_length=2, choices=Service.choices) + + # The text used to generate the embedding + text = models.TextField() + + # The result of the embedding operation + vectors = models.JSONField() + + # Which user does this embedding belong to? + user = models.ForeignKey(User, on_delete=models.CASCADE) + + def to_dict(self) -> dict: + """Convert the model to a dictionary.""" + return { + 'id': self.id, + 'created_at': self.created_at, + 'model': self.model, + 'service': self.service, + 'text': self.text, + 'vectors': self.vectors, + } diff --git a/chirps/embedding/providers/__init__.py b/chirps/embedding/providers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/chirps/embedding/providers/base.py b/chirps/embedding/providers/base.py new file mode 100644 index 00000000..37a7a4eb --- /dev/null +++ b/chirps/embedding/providers/base.py @@ -0,0 +1,17 @@ +"""Base implementation for embedding providers.""" +from abc import ABC +from typing import Any + +from django.contrib.auth.models import User + + +class EmbeddingError(Exception): + """Error class for the embedding providers.""" + + +class BaseEmbeddingProvider(ABC): + """Base implementation for embedding providers.""" + + def embed(self, user: User, model: str, text: str) -> Any: + """Generate embeddings for the specified text.""" + raise NotImplementedError diff --git a/chirps/embedding/providers/openai.py b/chirps/embedding/providers/openai.py new file mode 100644 index 00000000..9c6236db --- /dev/null +++ b/chirps/embedding/providers/openai.py @@ -0,0 +1,41 @@ +"""OpenAI implementation for embedding provider.""" +import logging +from typing import Any + +import openai +from django.contrib.auth.models import User + +from .base import BaseEmbeddingProvider, EmbeddingError + +logger = logging.getLogger(__name__) + + +class OpenAIEmbeddingProvider(BaseEmbeddingProvider): + """OpenAI implementation for embedding providers.""" + + def embed(self, user: User, model: str, text: str) -> Any: + """Use OpenAI to generate embeddings for the specified text.""" + # If the OpenAI API key is not set, raise an error + if user.profile.openai_key in [None, '']: + logger.error('User OpenAI key not set', extra={'user_id': user.id}) + raise EmbeddingError('OpenAI API key not set') + + # Initialize the OpenAI API key from the user's profile + openai.api_key = user.profile.openai_key + + logger.debug('Generating embedding for text', extra={'text': text}) + + # Generate the embedding + try: + response = openai.Embedding.create(model=model, input=text) + except openai.error.InvalidRequestError as err: + raise EmbeddingError(f'Embedding failure: {str(err)}') from err + + logger.debug( + 'Embedding complete', + extra={ + 'prompt_tokens': response['usage']['prompt_tokens'], + 'total_tokens': response['usage']['total_tokens'], + }, + ) + return response['data'][0]['embedding'] diff --git a/chirps/embedding/tests.py b/chirps/embedding/tests.py new file mode 100644 index 00000000..7938a516 --- /dev/null +++ b/chirps/embedding/tests.py @@ -0,0 +1,127 @@ +"""Tests for the embedding app.""" +# pylint: disable=consider-using-with +import json +from pathlib import Path +from unittest.mock import patch + +import openai +from django.test import TestCase +from django.urls import reverse + +from .models import Embedding + +# Build the path to the fixtures directory +fixture_path = Path(__file__).parent.resolve() / Path('./fixtures/embedding') + + +class TestEmbedding(TestCase): + """Test the embedding app.""" + + def setUp(self): + """Create a dummy user for testing.""" + username = 'test_user' + password = 'test_password' + email = 'test_user@mantiumai.com' + + # Create a new account (will automatically log us in) + self.client.post( + reverse('signup'), {'username': username, 'email': email, 'password1': password, 'password2': password} + ) + + # Set a dummy OpenAI key + self.client.post(reverse('profile'), {'openai_key': 'test_openai_key'}) + + def test_create_invalid(self): + """Test creating an embedding with invalid URL parameters.""" + # Missing text field should yield a 400 response + response = self.client.get(reverse('embedding_create'), {'model': 'test model', 'service': 'test service'}) + self.assertContains(response, text='"text": ["This field is required."]', status_code=400) + + # Missing model field should yield a 400 response + response = self.client.get(reverse('embedding_create'), {'text': 'test text', 'service': 'test service'}) + self.assertContains(response, text='"model": ["This field is required."]', status_code=400) + + # Missing service field should yield a 400 response + response = self.client.get(reverse('embedding_create'), {'text': 'test text', 'model': 'test model'}) + self.assertContains(response, text='"service": ["This field is required."]', status_code=400) + + # Pass in an invalid service ID + response = self.client.get( + reverse('embedding_create'), {'text': 'test text', 'model': 'test model', 'service': 'invalid service'} + ) + self.assertContains( + response, + text='"service": ["Select a valid choice. invalid service is not one of the available choices."]', + status_code=400, + ) + + @patch( + 'openai.Embedding.create', + side_effect=openai.error.InvalidRequestError('The model `invalid-model-001` does not exist', ''), + ) + def test_create_invalid_openai_model(self, _mock_openai_embedding_create): + """Pass in a junk model name to the OpenAI service.""" + response = self.client.get( + reverse('embedding_create'), {'text': 'test text', 'model': 'invalid-model-001', 'service': 'OA'} + ) + self.assertContains( + response, + text='{"error": "Embedding failure: The model `invalid-model-001` does not exist"}', + status_code=400, + ) + + @patch( + 'openai.Embedding.create', + return_value=json.loads(open(f'{fixture_path}/openai_embedding_create_mock.json', encoding='utf-8').read()), + ) + def test_create_valid(self, _mock_openai_embedding_create): + """Test creating a valid embedding.""" + # Verify there are no embeddings in the database + self.assertEqual(0, Embedding.objects.all().count()) + + # Fire off an embedding request that will hit the OpenAI mock + # The request SHOULD create a new Embedding object in the database + response = self.client.get( + reverse('embedding_create'), {'text': 'test text', 'model': 'invalid-model-001', 'service': 'OA'} + ) + self.assertContains(response, text='-0.016634132713079453', count=2, status_code=200) + + # There now should be one embedding in the database + self.assertEqual(1, Embedding.objects.all().count()) + + # Fire off a duplicate request - verify there is still only one model in the DB + response = self.client.get( + reverse('embedding_create'), {'text': 'test text', 'model': 'invalid-model-001', 'service': 'OA'} + ) + self.assertContains(response, text='-0.016634132713079453', count=2, status_code=200) + self.assertEqual(1, Embedding.objects.all().count()) + + @patch( + 'openai.Embedding.create', + return_value=json.loads(open(f'{fixture_path}/openai_embedding_create_mock.json', encoding='utf-8').read()), + ) + def test_delete(self, _mock_openai_embedding_create): + """Test deleting an embedding.""" + # Verify there are no embeddings in the database + self.assertEqual(0, Embedding.objects.all().count()) + + # Fire off an embedding request that will hit the OpenAI mock + # The request SHOULD create a new Embedding object in the database + response = self.client.get( + reverse('embedding_create'), {'text': 'test text', 'model': 'invalid-model-001', 'service': 'OA'} + ) + self.assertContains(response, text='-0.016634132713079453', count=2, status_code=200) + + # List all of the embeddings (there should be only one) + response = self.client.get(reverse('embedding_list')) + json_data = response.json() + self.assertEqual(len(json_data['embeddings']), 1) + + # Delete the embedding + response = self.client.get( + reverse('embedding_delete', kwargs={'embedding_id': json_data['embeddings'][0]['id']}) + ) + self.assertEqual(response.status_code, 200) + + # Verify the embedding is deleted from the database + self.assertEqual(0, Embedding.objects.all().count()) diff --git a/chirps/embedding/urls.py b/chirps/embedding/urls.py new file mode 100644 index 00000000..0cefc08a --- /dev/null +++ b/chirps/embedding/urls.py @@ -0,0 +1,10 @@ +"""URLs for the embedding app.""" +from django.urls import path + +from . import views + +urlpatterns = [ + path('', views.show, name='embedding_list'), + path('create/', views.create, name='embedding_create'), + path('delete//', views.delete, name='embedding_delete'), +] diff --git a/chirps/embedding/utils.py b/chirps/embedding/utils.py new file mode 100644 index 00000000..77afcf11 --- /dev/null +++ b/chirps/embedding/utils.py @@ -0,0 +1,36 @@ +"""Utility functions for the embedding app.""" +from django.contrib.auth.models import User + +from .models import Embedding +from .providers.base import BaseEmbeddingProvider + + +def create_embedding(text: str, model: str, service: str, user: User) -> Embedding: + """Pull an embedding from the database, or generate a new one.""" + # Search for the text to see if an embedding already exists + try: + embedding = Embedding.objects.get( + text=text, + model=model, + service=service, + user=user, + ) + except Embedding.DoesNotExist: # Oh noes! We need to generate a new embedding. + # Fetch the required provider class from the name of the service requested by the user. The service name + # should be one of the enum values found in Embedding.Service. + provider: BaseEmbeddingProvider = Embedding.Service.get_provider_from_service_name(service) + + # Use the specified service to generate embeddings with the specified model + # Raises EmbeddingError if the embedding fails + embed_result = provider.embed(user=user, model=model, text=text) + + # Save the embedding result to the database + embedding = Embedding.objects.create( + model=model, + service=service, + text=text, + vectors=embed_result, + user=user, + ) + + return embedding diff --git a/chirps/embedding/views.py b/chirps/embedding/views.py new file mode 100644 index 00000000..60945684 --- /dev/null +++ b/chirps/embedding/views.py @@ -0,0 +1,54 @@ +"""Views for the embedding app.""" +import json + +from django.contrib.auth.decorators import login_required +from django.core.paginator import Paginator +from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse +from django.shortcuts import get_object_or_404 + +from .forms import CreateEmbeddingForm +from .models import Embedding +from .providers.base import EmbeddingError +from .utils import create_embedding + + +@login_required +def create(request): + """Create a new embedding.""" + # Verify the reqquest has the requred URL parameters + form = CreateEmbeddingForm(request.GET) + if not form.is_valid(): + return HttpResponseBadRequest(json.dumps(form.errors), status=400) + + # Use the specified service to generate embeddings with the specified model + try: + embedding: Embedding = create_embedding( + text=form.cleaned_data['text'], + model=form.cleaned_data['model'], + service=form.cleaned_data['service'], + user=request.user, + ) + except EmbeddingError as err: + return HttpResponseBadRequest(json.dumps({'error': str(err)}), status=400) + + # Return the vectors and it's siesta time. + return JsonResponse({'embedding': embedding.vectors}) + + +@login_required +def delete(request, embedding_id): + """Delete an existing embedding.""" + get_object_or_404(Embedding, pk=embedding_id, user=request.user).delete() + return HttpResponse('OK', status=200) + + +@login_required +def show(request): + """List all embeddings.""" + # Paginate the number of items returned to the user, defaulting to 25 per page + user_embeddings = Embedding.objects.filter(user=request.user).order_by('id') + paginator = Paginator(user_embeddings, request.GET.get('item_count', 25)) + page_number = request.GET.get('page') + page_obj = paginator.get_page(page_number) + + return JsonResponse({'embeddings': [embedding.to_dict() for embedding in page_obj.object_list]}) diff --git a/requirements.txt b/requirements.txt index 52f41f72..07fa5122 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ django-celery-results==2.5.1 django-fernet-fields==0.6 django-polymorphic==3.1.0 mantium-client +openai==0.27.8 redis==4.5.5 requests==2.31.0 pinecone-client