diff --git a/fsgp/README.md b/fsgp/README.md new file mode 100644 index 0000000..1cbcb28 --- /dev/null +++ b/fsgp/README.md @@ -0,0 +1,17 @@ +# FSGP Usage + +#### 1. Make sure you're in a virtual environment and have all the requirements installed: + +``` +virtualenv venv --python=python3.10 +./venv/Scripts/activate +pip install -r requirements.txt +``` + +#### 2. Run the ETL script. The script will guide you through its usage: + +``` +python etl.py +``` + +That's it! \ No newline at end of file diff --git a/fsgp/data/TrackData.csv b/fsgp/data/TrackData.csv new file mode 100644 index 0000000..0ab9941 --- /dev/null +++ b/fsgp/data/TrackData.csv @@ -0,0 +1,12 @@ +WKT,name,description +"LINESTRING (-86.3685966 37.0010537, -86.3685242 37.0010848, -86.3684383 37.0011222, -86.3683485 37.0011629, -86.3682533 37.0012047, -86.368154 37.0012497, -86.3680695 37.0012882, -86.3679837 37.00133, -86.3678912 37.0013696, -86.3678013 37.0014114, -86.3677155 37.0014521, -86.3676364 37.0014939, -86.3675478 37.0015367, -86.3674566 37.0015785, -86.3673573 37.0016234, -86.3672514 37.0016706, -86.3671521 37.0017166, -86.3670582 37.0017605, -86.3669617 37.0018055, -86.3668544 37.0018548, -86.366748 37.0019014, -86.3666528 37.0019432, -86.3665629 37.0019893, -86.3664717 37.0020332, -86.3663712 37.0020771, -86.3662733 37.0021221, -86.3661767 37.0021681, -86.3660855 37.002212, -86.3660124 37.002244, -86.365932 37.002275, -86.3658488 37.0023125, -86.3657871 37.0023425, -86.3657268 37.0023725, -86.3656235 37.0024207, -86.3655162 37.0024689, -86.3654183 37.0025171, -86.365307 37.0025663, -86.3651984 37.0026177, -86.3650949 37.0026652, -86.3649863 37.0027198, -86.364891 37.0027637, -86.3647864 37.0028119, -86.3646885 37.0028591, -86.3646054 37.0028998, -86.3645029 37.0029479, -86.3644063 37.0029951, -86.3643245 37.0030325, -86.3642333 37.0030775, -86.3641582 37.0031139, -86.3640777 37.0031536, -86.3639892 37.0031964, -86.3639074 37.003235, -86.3638373 37.0032686)",1, +"LINESTRING (-86.3637706 37.0032972, -86.3636888 37.0033346, -86.3636016 37.0033711, -86.3635185 37.0034075, -86.3634415 37.0034423, -86.3633651 37.0034776, -86.3632926 37.0035141, -86.3632283 37.0035494, -86.3631599 37.0035826, -86.3630928 37.0036158, -86.3630043 37.0036554, -86.3629091 37.0036979, -86.3628367 37.0037343, -86.3627616 37.0037718, -86.3626878 37.0038082, -86.3626181 37.0038414, -86.3625282 37.0038789, -86.3624477 37.0039164, -86.3623726 37.0039507, -86.3622922 37.003986, -86.3622171 37.0040203, -86.3621326 37.0040556, -86.3620588 37.0040921, -86.361969 37.0041317, -86.3618912 37.0041735, -86.3618107 37.0042152, -86.3617316 37.0042484, -86.3616484 37.0042848, -86.3615653 37.0043234, -86.3614929 37.0043598, -86.361407 37.0043973, -86.361313 37.004443, -86.3612245 37.0044826, -86.3611293 37.0045254, -86.3610247 37.0045736, -86.3609362 37.0046165, -86.360849 37.0046604, -86.3607538 37.0047064, -86.3606774 37.0047428, -86.3605875 37.0047942, -86.36055 37.0048285, -86.3605164 37.0048735, -86.3604977 37.0049292)",2a, +"LINESTRING (-86.3638004 37.0032971, -86.3637668 37.0033324, -86.3637373 37.0033763, -86.3637145 37.0034267, -86.363689 37.0034856, -86.3636676 37.0035445, -86.3636475 37.0036109, -86.3636327 37.0036634, -86.363618 37.0037266, -86.3636019 37.0037972, -86.3635844 37.0038615, -86.3635657 37.0039333, -86.3635522 37.0039975, -86.3635402 37.0040575, -86.3635254 37.0041239, -86.363512 37.0041796, -86.3634999 37.004216, -86.3634839 37.0042481, -86.3634637 37.0042813, -86.3634369 37.0043102, -86.3634061 37.0043413, -86.3633752 37.0043756, -86.3633149 37.0044238, -86.3632505 37.0044634, -86.3631794 37.0045148, -86.3631097 37.0045641, -86.3630252 37.0046112, -86.3629474 37.0046508, -86.3628817 37.004689, -86.3628415 37.0047168, -86.3628147 37.0047436, -86.3627852 37.0047746, -86.362757 37.0048046, -86.3627355 37.00484, -86.3627127 37.0048882, -86.362706 37.0049481, -86.362706 37.0050113, -86.3627154 37.005067, -86.3627235 37.0051152, -86.3627262 37.005172, -86.3627181 37.005233, -86.3627114 37.0052994, -86.3627007 37.005354, -86.3626779 37.0054215, -86.362647 37.0054826, -86.3626028 37.0055372, -86.3625518 37.0055854, -86.3624995 37.0056271, -86.3624392 37.0056636, -86.3623654 37.0056925, -86.3622916 37.0057139, -86.3622125 37.0057289, -86.3621266 37.0057328, -86.3620421 37.0057296, -86.361967 37.00572, -86.3618973 37.005706, -86.3618248 37.0056793, -86.3617538 37.0056471, -86.361688 37.0056107, -86.3616465 37.0055657, -86.3615875 37.0055023, -86.3615432 37.0054337, -86.3615177 37.0053716, -86.3614963 37.0052956, -86.3614936 37.0052163, -86.3615057 37.0051424, -86.3615445 37.0050653, -86.3615915 37.0049839, -86.3616305 37.004924, -86.3616707 37.0048629, -86.3617123 37.0048072, -86.3617552 37.0047494, -86.3617861 37.0046894, -86.3618035 37.0046327, -86.3618129 37.0045812, -86.3618142 37.0045213, -86.3618041 37.0044495, -86.3617867 37.0043906, -86.3617572 37.0043285, -86.3617303 37.0042814, -86.3616955 37.0042364, -86.3616526 37.0041903, -86.3615801 37.0041411, -86.3615064 37.0041068, -86.3614326 37.0040747, -86.3613669 37.0040575, -86.3612824 37.0040436, -86.3611805 37.0040425, -86.3610826 37.00405, -86.360982 37.0040672, -86.3609056 37.0041004, -86.3608372 37.0041368, -86.360775 37.0041794, -86.3607294 37.0042211, -86.3606771 37.0042715, -86.3606422 37.004325, -86.3606114 37.0043882, -86.3605886 37.0044589, -86.3605723 37.0045354, -86.3605576 37.0046061, -86.3605415 37.0046811, -86.3605267 37.0047539, -86.360516 37.0048139, -86.3605012 37.0048814, -86.3604892 37.0049338)",2b +, +"LINESTRING (-86.3604883 37.0049698, -86.3604802 37.0050255, -86.3604682 37.0050865, -86.3604507 37.005155, -86.3604346 37.0052129, -86.3604212 37.0052782, -86.3604118 37.0053435, -86.3603971 37.0054153, -86.3603823 37.0054903, -86.3603703 37.0055567, -86.3603568 37.0056209, -86.3603568 37.0056852, -86.3603689 37.0057452, -86.3604024 37.0058137, -86.360432 37.0058812, -86.3604655 37.0059476, -86.3604869 37.0059861, -86.3605285 37.0060204, -86.3606385 37.0060536, -86.3607324 37.0060857, -86.3608222 37.0061221, -86.3609215 37.0061639, -86.3610033 37.0062035, -86.3610891 37.0062421, -86.3611615 37.0062774, -86.3612456 37.0063079, -86.361326 37.0063358, -86.3614105 37.0063604, -86.3614856 37.0063797, -86.3615782 37.0064022, -86.361668 37.0064182, -86.3617646 37.0064343, -86.3618638 37.0064418, -86.3619577 37.0064461, -86.3620569 37.006445, -86.3621629 37.0064386, -86.3622702 37.0064279, -86.3623708 37.0064161, -86.362466 37.0063925, -86.3625599 37.006369, -86.3626618 37.00634, -86.3627583 37.0063079, -86.3628428 37.0062661, -86.3629313 37.0062233, -86.3630078 37.0061805, -86.3630923 37.0061323, -86.3631674 37.0060819, -86.3632438 37.006023, -86.3633002 37.0059738, -86.3633551 37.0059127, -86.3634155 37.0058527, -86.3634557 37.0057896, -86.363496 37.0057221, -86.3635389 37.0056546, -86.3635818 37.0055807, -86.363618 37.0055047, -86.3636462 37.0054426, -86.3636716 37.0053751, -86.3637025 37.0053023, -86.363728 37.0052316, -86.3637575 37.0051641, -86.363791 37.0050924, -86.3638272 37.0050163, -86.3638795 37.0049424, -86.363917 37.0048739, -86.3639653 37.0048064, -86.3640391 37.004739, -86.3640901 37.0046822, -86.3641652 37.0046254, -86.3642561 37.0045655, -86.364342 37.0045162)",3, +"LINESTRING (-86.3644082 37.0044888, -86.3644873 37.0044566, -86.3645933 37.0044149, -86.3646845 37.0043827, -86.364781 37.0043474, -86.3648736 37.0043131, -86.3649661 37.0042756, -86.3650626 37.0042392, -86.3651592 37.0042017, -86.3652558 37.0041653, -86.365347 37.0041268, -86.3654475 37.0040871, -86.3655535 37.0040475, -86.3656433 37.0040132, -86.3657426 37.0039865, -86.3658445 37.003964, -86.3659397 37.00395, -86.366039 37.0039425, -86.3661449 37.0039415, -86.3662442 37.0039468, -86.3663541 37.0039533, -86.3664534 37.003964, -86.3665486 37.0039811, -86.3666398 37.0040079)",4a, +"LINESTRING (-86.3643876 37.0044802, -86.3644453 37.0044298, -86.3645123 37.0043731, -86.3645767 37.0043131, -86.3646424 37.0042563, -86.3647041 37.0042007, -86.3647618 37.0041439, -86.3648261 37.004085, -86.3648919 37.004025, -86.3649455 37.003964, -86.3650085 37.0038986, -86.3650649 37.0038504, -86.3651306 37.0037958, -86.3651882 37.003738, -86.3652553 37.0036769, -86.365321 37.003617, -86.3653827 37.0035591, -86.3654498 37.0034991, -86.3655168 37.0034413, -86.3655906 37.003376, -86.3656549 37.0033053, -86.3657341 37.0032357, -86.3658011 37.0031736, -86.3658722 37.0031082, -86.3659419 37.0030408, -86.3660197 37.0029711, -86.3660988 37.0029026, -86.3661686 37.0028437, -86.366245 37.0027848, -86.3663376 37.0027291, -86.3664307 37.002681, -86.3665219 37.0026371, -86.3666118 37.0025942, -86.3667097 37.0025535, -86.3668049 37.0025203, -86.366888 37.0024818, -86.3669739 37.00244, -86.3670637 37.0024004, -86.3671495 37.002364, -86.3672246 37.0023275, -86.3673241 37.0022837, -86.3674112 37.002243, -86.3675051 37.0022044, -86.367595 37.0021658, -86.3676808 37.0021251, -86.3677733 37.0020823, -86.367886 37.0020266, -86.3679892 37.0019816, -86.3680831 37.0019399, -86.3681783 37.0018981, -86.3682695 37.0018574, -86.3683487 37.0018199, -86.3684345 37.0017792, -86.3685096 37.0017449, -86.3685619 37.0017214, -86.3686169 37.0017021, -86.3686665 37.0016924, -86.3687429 37.0016817, -86.3688449 37.0016785, -86.3689213 37.0016796, -86.3689951 37.0016796, -86.369089 37.0016785, -86.3691708 37.0016785, -86.3692378 37.0016796, -86.369321 37.0016785, -86.3694189 37.0016796, -86.3694886 37.0016828, -86.3695838 37.0016839, -86.3696844 37.0016892, -86.3697877 37.0016946, -86.3698748 37.0016978, -86.3699392 37.001701, -86.3700009 37.0016999, -86.3700411 37.0016914, -86.3700948 37.0016732, -86.3701417 37.0016443, -86.3701994 37.0015918, -86.3702651 37.0015286, -86.3703362 37.0014729, -86.3704099 37.0014043, -86.370485 37.0013347, -86.3705521 37.001263, -86.3706218 37.0012008, -86.3706701 37.0011644, -86.3707211 37.0011291, -86.3707787 37.0010959, -86.3708713 37.0010541, -86.3709652 37.0010156, -86.371059 37.0009759, -86.3711623 37.0009309, -86.3712669 37.000887, -86.3713715 37.0008442, -86.3714573 37.0008067, -86.3715432 37.000766, -86.3716263 37.0007307, -86.3717108 37.0006942, -86.3717792 37.0006621, -86.3718557 37.0006321, -86.3719039 37.000615, -86.3719482 37.0005968, -86.3720005 37.0005871, -86.3720407 37.0005882, -86.3720863 37.0005946, -86.3721252 37.0006096, -86.3721668 37.0006321, -86.3721909 37.0006578, -86.3722097 37.0006942, -86.3722164 37.0007274, -86.3722204 37.0007564, -86.3722204 37.0007821, -86.3722191 37.0008335, -86.3722191 37.0008785, -86.3722124 37.0009384, -86.372207 37.0010016, -86.3721976 37.0010659, -86.3721789 37.0011216, -86.3721587 37.0011762, -86.3721333 37.0012212, -86.372093 37.0012833, -86.3720515 37.0013401, -86.3719944 37.0014012, -86.3719381 37.0014537, -86.3718737 37.0015008, -86.3718174 37.0015404, -86.3717476 37.0015843, -86.37169 37.0016207, -86.3716336 37.0016593, -86.3715666 37.0016946, -86.3714968 37.0017289, -86.3714258 37.0017685, -86.3713493 37.0018028, -86.3712756 37.0018328, -86.3711937 37.0018639, -86.3711213 37.0018906, -86.3710449 37.0019174, -86.3709577 37.001942, -86.3708799 37.0019667, -86.3707968 37.0019924, -86.3707203 37.002017, -86.3706426 37.0020395, -86.3705728 37.0020641, -86.3705058 37.0020866, -86.3704374 37.0021188, -86.3703556 37.0021541, -86.3702845 37.002197, -86.3702054 37.0022419, -86.3701463 37.0022858, -86.3700927 37.0023255, -86.3700417 37.0023715, -86.3699868 37.0024251, -86.3699358 37.0024765, -86.3698902 37.0025343, -86.3698433 37.0025889, -86.3698044 37.0026446, -86.3697641 37.0027068, -86.3697293 37.0027764, -86.3696998 37.0028353, -86.3696743 37.0028953, -86.3696475 37.0029627, -86.3696153 37.0030248, -86.3695791 37.0030848, -86.3695593 37.0031364, -86.3695312 37.0031825, -86.3695017 37.0032157, -86.3694695 37.0032553, -86.3694212 37.0032981, -86.3693662 37.0033378, -86.3693139 37.0033752, -86.3692603 37.0033988, -86.3692026 37.0034181, -86.3691315 37.0034384, -86.3690578 37.0034502, -86.3689961 37.0034556, -86.3689371 37.0034545, -86.3688566 37.0034491, -86.3688123 37.0034406, -86.3687453 37.0034224, -86.3686849 37.0033999, -86.3686045 37.003371, -86.3685347 37.0033217, -86.3684422 37.0032703, -86.3683564 37.0032167, -86.3682799 37.0031696, -86.3681914 37.0031182, -86.3681082 37.0030689, -86.3680305 37.0030261, -86.3679554 37.0029843, -86.367891 37.0029661, -86.3678186 37.002949, -86.3677475 37.0029425, -86.3676751 37.0029415, -86.3676174 37.0029447, -86.3675503 37.0029543, -86.3674779 37.0029704, -86.3674122 37.002994, -86.3673385 37.0030197, -86.3672513 37.0030561, -86.3671735 37.0030871, -86.3671011 37.0031246, -86.3670354 37.0031653, -86.366975 37.0032049, -86.366912 37.0032596, -86.3668476 37.0033078, -86.3667815 37.0033607, -86.3667238 37.0034099, -86.3666541 37.0034624, -86.3665978 37.0035117, -86.3665495 37.0035545, -86.3665133 37.0035866, -86.3664932 37.0036156, -86.3664757 37.003653, -86.366461 37.0036937, -86.3664543 37.0037409, -86.3664623 37.0037912, -86.3664744 37.0038426, -86.3664999 37.0038865, -86.366528 37.0039272, -86.3665683 37.0039615, -86.3666192 37.0039893, -86.3666528 37.0040086)",4b, +"LINESTRING (-86.3667177 37.0040307, -86.3668021 37.0040575, -86.3669027 37.0040907, -86.3669818 37.0041153, -86.3670637 37.0041442, -86.3671522 37.0041688, -86.367238 37.0041967, -86.3673319 37.0042245, -86.3674311 37.004262, -86.367525 37.0042888, -86.3676175 37.0043177, -86.3676926 37.0043434, -86.3677677 37.0043702, -86.3678509 37.0043938, -86.3679327 37.0044162, -86.3680172 37.0044334, -86.368103 37.0044452, -86.3681888 37.0044537, -86.3682841 37.0044623, -86.368386 37.0044655, -86.3684986 37.0044634, -86.3685898 37.0044548, -86.3686837 37.0044452, -86.3687722 37.0044259, -86.3688809 37.0044023, -86.3689935 37.0043734, -86.3690686 37.0043381, -86.3691665 37.004307, -86.3692489 37.0042677, -86.3693468 37.0042249, -86.3694246 37.0041692, -86.3695024 37.0041114, -86.3695721 37.0040535, -86.3696419 37.0039978, -86.3697116 37.0039346, -86.3697773 37.0038736, -86.369839 37.0038104, -86.3699074 37.0037451, -86.3699651 37.0036894, -86.3700227 37.0036358, -86.3700858 37.0035759, -86.3701448 37.0035137, -86.3701957 37.0034548, -86.3702494 37.0033831, -86.3703044 37.0033145, -86.370358 37.0032503, -86.3704103 37.0031742, -86.3704707 37.0031014, -86.3705243 37.0030361, -86.3705806 37.0029697, -86.3706504 37.002899, -86.3707228 37.0028326, -86.3707898 37.0027694, -86.3708636 37.002703, -86.3709468 37.0026366, -86.3710165 37.0025841, -86.3710862 37.0025348, -86.3711721 37.0024877, -86.3712458 37.002447, -86.3713345 37.0024028, -86.3714365 37.0023482, -86.371521 37.0023075, -86.3716028 37.0022636, -86.3716899 37.0022175, -86.3717637 37.0021704, -86.3718442 37.002134, -86.3719179 37.0020944, -86.3719863 37.0020504, -86.372052 37.0020108, -86.3721218 37.0019615, -86.3721808 37.0019198, -86.3722438 37.001878, -86.3723001 37.0018309, -86.3723538 37.0017838, -86.3724168 37.0017345, -86.3724812 37.0016777, -86.3725375 37.0016295, -86.3725885 37.0015824, -86.3726515 37.0015235, -86.3726984 37.0014699, -86.3727481 37.0014228, -86.3728084 37.001365, -86.3728621 37.0013071, -86.3729197 37.0012407, -86.3729814 37.001184, -86.3730377 37.0011251, -86.373105 37.0010505, -86.3731747 37.0009766, -86.3732391 37.000908, -86.3732967 37.0008405, -86.3733598 37.0007795, -86.3734134 37.0007163, -86.3734765 37.000652, -86.3735355 37.000591, -86.3735837 37.0005342, -86.3736387 37.0004775, -86.3736951 37.0004196, -86.3737474 37.0003639, -86.373805 37.0003029, -86.373856 37.0002483, -86.3739163 37.000184, -86.37397 37.0001337, -86.3740209 37.0000747, -86.3740773 37.0000094, -86.3741309 36.9999516, -86.3741953 36.999898, -86.3742369 36.9998423, -86.374265 36.9998027, -86.3742918 36.9997588, -86.3743079 36.9997106, -86.3743227 36.9996656, -86.3743334 36.999611, -86.3743334 36.9995596, -86.3743281 36.9995146, -86.3743187 36.9994578, -86.3743053 36.9993989, -86.3742945 36.9993314, -86.3742768 36.9992567, -86.374262 36.9991817, -86.3742432 36.9991132, -86.3742218 36.999066, -86.3741936 36.9990339, -86.3741628 36.999006, -86.3741252 36.99899, -86.3740743 36.9989707, -86.3740059 36.9989632, -86.3739576 36.9989664, -86.3738825 36.9989803, -86.3738033 36.99899, -86.3737054 36.9990082, -86.3736143 36.9990275, -86.373507 36.9990489, -86.3734225 36.9990639, -86.3733326 36.9990799, -86.3732495 36.9990971, -86.3731489 36.9991142, -86.3730671 36.9991281, -86.3729813 36.9991453, -86.3728914 36.9991624, -86.3727922 36.9991817, -86.3726996 36.9991956, -86.3725937 36.9992181, -86.372481 36.9992428, -86.3723778 36.999276, -86.3722946 36.9993113, -86.3722302 36.9993413, -86.3721618 36.9993702, -86.3720814 36.9994098, -86.3720224 36.999442)",5, +"LINESTRING (-86.3719867 36.9994742, -86.3719478 36.9995128, -86.3719156 36.9995535, -86.3718727 36.9995995, -86.3718419 36.9996391, -86.371803 36.9996873, -86.3717587 36.9997409, -86.3717212 36.9997955, -86.3716836 36.9998459, -86.3716501 36.9998898, -86.3716152 36.9999444, -86.3715736 37.0000087, -86.3715307 37.0000643, -86.3714932 37.0001179, -86.3714476 37.00018, -86.3714181 37.0002218, -86.3713899 37.0002571, -86.3713604 37.0002893, -86.3713255 37.0003171, -86.3712933 37.0003407, -86.3712531 37.00036, -86.3712102 37.000376, -86.3711619 37.000391, -86.3711083 37.0004049, -86.37106 37.0004124, -86.3710171 37.0004124, -86.370954 37.0004124, -86.3708924 37.0004017, -86.3708374 37.0003867, -86.3707784 37.0003632, -86.3707033 37.0003385, -86.3706295 37.000316, -86.370549 37.0002925, -86.3704887 37.0002882, -86.3704082 37.000285, -86.3703264 37.0002839, -86.3702366 37.0002914, -86.3701628 37.0003128, -86.3701065 37.000345)",6a, +"LINESTRING (-86.371974 36.9994654, -86.3718976 36.9994944, -86.3718211 36.9995286, -86.371746 36.9995618, -86.3716495 36.9996057, -86.3715542 36.9996475, -86.3714577 36.9997043, -86.3713678 36.9997493, -86.37129 36.9997889, -86.3711734 36.999836, -86.3710714 36.9998853, -86.3709722 36.9999303, -86.3708663 36.9999785, -86.3707791 37.0000224, -86.3706785 37.0000695, -86.3705913 37.0001113, -86.3704975 37.0001509, -86.3704143 37.0001863, -86.3703218 37.0002291, -86.3702453 37.0002666, -86.3701407 37.0003169, -86.3700884 37.0003469)",6b, +"LINESTRING (-86.3700436 37.0003717, -86.3699645 37.000406, -86.36988 37.0004456, -86.3697875 37.0004874, -86.3696762 37.0005399, -86.3695742 37.000586, -86.3694924 37.0006277, -86.3693972 37.0006706, -86.3693221 37.000707, -86.3692417 37.0007466, -86.3691558 37.0007841, -86.3690821 37.0008205, -86.3690043 37.0008569, -86.3689292 37.0008923, -86.3688594 37.0009244, -86.3687991 37.0009544, -86.368724 37.0009919, -86.3686489 37.0010272, -86.3685966 37.0010537)",7, diff --git a/fsgp/data/TrackDataTest.csv b/fsgp/data/TrackDataTest.csv new file mode 100644 index 0000000..4dbf263 --- /dev/null +++ b/fsgp/data/TrackDataTest.csv @@ -0,0 +1,5 @@ +WKT,name,description +"LINESTRING (50 50, 51 51)",1, +"LINESTRING (1 1, 2 2, 3 3)",2, +,, +"LINESTRING (4 4, 5 5)",3, diff --git a/fsgp/fsgp.py b/fsgp/fsgp.py new file mode 100644 index 0000000..9d8d9b9 --- /dev/null +++ b/fsgp/fsgp.py @@ -0,0 +1,174 @@ +import sys +from pathlib import Path + +import psycopg2 +import questionary +from termcolor import colored +import fsgp_routemodel.fsgp_routemodel as fsgp + + +def validate_db_creds(): + answers = questionary.form( + db_host=questionary.text("Database host:port", default=""), + db_name=questionary.text("Database name", default=""), + db_user=questionary.text("Database user", default=""), + db_password=questionary.password("Database user password", default=""), + ).ask() + db_user = answers["db_user"] + db_password = answers["db_password"] + db_host = answers["db_host"] + db_name = answers["db_name"] + host, port = db_host.split(":") + try: + conn = psycopg2.connect( + host=host, port=port, user=db_user, password=db_password, dbname=db_name + ) + conn.close() + return True, db_user, db_password, db_host, db_name + except Exception as e: + print(colored(e, "red")) + return False, None, None, None, None + + +def validate_path(path, extension): + path = Path(path).absolute() + return path.is_file() and path.suffix == extension + + +def cmd_routemodel_import(): + answers = questionary.form( + track_data_filepath=questionary.path( + "Path to track data (csv file)", + default="", + validate=lambda p: validate_path(p, ".csv"), + ), + confirm=questionary.confirm( + "Confirm import operation", + default=False, + auto_enter=False, + ), + ).ask() + track_data_filepath = answers["track_data_filepath"] + confirm = answers["confirm"] + + if not confirm: + print(colored("fsgp track data import cancelled", "red")) + else: + fsgp.import_track_segments(track_data_filepath) + print(colored("fsgp track data imported", "green")) + + +def cmd_routemodel_construct(): + answers = questionary.form( + route_name=questionary.text("Name of route"), + segment_data_filepath=questionary.path( + "Path to segment data (json file)", + default="", + validate=lambda p: validate_path(p, ".json"), + ), + segment_order=questionary.text("Order of track segments"), + num_loops=questionary.text("Number of loops"), + confirm=questionary.confirm( + "Confirm import operation", + default=False, + auto_enter=False, + ), + ).ask() + segment_data_filepath = answers["segment_data_filepath"] + segment_order = answers["segment_order"] + num_loops = answers["num_loops"] + route_name = answers["route_name"] + confirm = answers["confirm"] + + if not confirm: + print(colored("fsgp routemodel constuctions cancelled", "red")) + else: + fsgp.construct_route( + segment_data_filepath, segment_order, int(num_loops), route_name + ) + print(colored("fsgp routemodel constructed", "green")) + + +def cmd_routemodel_seed(db_user, db_password, db_host, db_name): + answers = questionary.form( + route_file_path=questionary.path( + "Path to route data (json file)", + default="", + validate=lambda p: validate_path(p, ".json"), + ), + confirm=questionary.confirm( + "Confirm import operation", + default=False, + auto_enter=False, + ), + ).ask() + route_file_path = answers["route_file_path"] + confirm = answers["confirm"] + + if not confirm: + print(colored("fsgp database seed cancelled", "red")) + else: + print("1) Parsing fsgp routemodel json format to csv format...") + csv_filepath = fsgp.route_json_to_gdf(route_file_path) + print("2) Seeding fsgp routemodel csv into database...") + fsgp.seed_from_csv(csv_filepath, db_user, db_password, db_host, db_name) + print(colored("fsgp database seed completed", "green")) + + +def route_model(db_user, db_password, db_host, db_name): + while True: + route_model_option = questionary.select( + "Select FSGP routemodel operation", + choices=[ + "Import Track Data", + "Construct Track Routemodel", + "Seed Database", + "Exit", + ], + ).ask() + + if route_model_option == "Import Track Data": + cmd_routemodel_import() + elif route_model_option == "Construct Track Routemodel": + cmd_routemodel_construct() + elif route_model_option == "Seed Database": + cmd_routemodel_seed(db_user, db_password, db_host, db_name) + else: + break + + +if __name__ == "__main__": + print( + colored( + "Using this ETL script requires your database to be updated and working with db_gateway", + "yellow", + ) + ) + + db_user, db_password, db_host, db_name = None, None, None, None + auth, db_user, db_password, db_host, db_name = validate_db_creds() + if auth is False: + print(colored("Incorrect database credentials", "red")) + sys.exit(1) + + etl_name = questionary.select( + "Select FSGP ETL operation", + choices=[ + "routemodel", + "location_service", + "weather", + "speed_limit/street_names", + "drop_tables", + ], + ).ask() + + if etl_name == "routemodel": + route_model(db_user, db_password, db_host, db_name) + elif etl_name == "location_service": + print(colored("NOT IMPLEMENTED", "red")) + elif etl_name == "weather": + print(colored("NOT IMPLEMENTED", "red")) + elif etl_name == "speed_limit/street_names": + print(colored("NOT IMPLEMENTED", "red")) + elif etl_name == "drop_tables": + print(colored("NOT IMPLEMENTED", "red")) diff --git a/fsgp/fsgp_routemodel/fsgp_routemodel.py b/fsgp/fsgp_routemodel/fsgp_routemodel.py new file mode 100644 index 0000000..8102dee --- /dev/null +++ b/fsgp/fsgp_routemodel/fsgp_routemodel.py @@ -0,0 +1,122 @@ +import pandas as pd +import numpy as np +import json +from pathlib import Path +from sqlalchemy import create_engine, inspect, String +import geopandas as gpd +from geoalchemy2 import Geometry +from geopy import distance + + +def import_track_segments(track_data_filepath): + file = Path(track_data_filepath) + if not file.is_file(): + raise FileNotFoundError("No file exists at the location specified") + + segments = {} + dtypes = {"WKT": str, "name": str, "description": str} + df = pd.read_csv(file, dtype=dtypes) + df.dropna(subset=["WKT"], inplace=True) + for _, row in df.iterrows(): + linestring, name = row["WKT"], row["name"] + linestring = linestring[12:-1] + points = [] + for p in linestring.split(","): + p = p.strip().split(" ") + lng, lat = p[0], p[1] + points.append({"lng": float(lng), "lat": float(lat)}) + segments[name] = points + + file_path = file.parent / f"{file.stem}_segments.json" + with open(file_path, "w") as f: + json.dump(segments, f) + return file_path + + +def construct_route(segments_data_filepath, segment_order, num_loops, route_name): + file = Path(segments_data_filepath) + if not file.is_file(): + raise FileNotFoundError("No file exists at the location specified") + + segments = {} + with open(file, "r") as f: + segments = json.load(f) + + segment_order = segment_order.split(" ") + route_points = [segments[o] for o in segment_order] + route_points = [point for rps in route_points * num_loops for point in rps] + route = {"points": route_points} + + file_path = file.parent / f"{route_name}.json" + with open(file_path, "w") as f: + json.dump(route, f) + return file_path + + +def route_json_to_gdf(route_file_path): + file = Path(route_file_path) + if not file.is_file(): + raise FileNotFoundError("No file exists at the location specified") + route_json = {} + gdf_rows = [] + with open(file, "r") as f: + route_json = json.load(f) + for point in route_json["points"]: + lon = point.get("lng", 0) + lat = point.get("lat", 0) + data = { + "lon": lon, + "lat": lat, + "geo": "POINT({} {})".format(lon, lat), + "geopy_elapsed_dist_m": 0, + "geopy_dist_from_last_m": 0, + } + if len(gdf_rows) > 0: + prev = gdf_rows[-1] + prev_coor = (prev["lat"], prev["lon"]) + curr_coor = (data["lat"], data["lon"]) + dist = distance.great_circle(prev_coor, curr_coor).m + data["geopy_dist_from_last_m"] = dist + data["geopy_elapsed_dist_m"] = prev["geopy_elapsed_dist_m"] + dist + gdf_rows.append(data) + + gdf = gpd.GeoDataFrame(gdf_rows) + csv_filepath = file.with_suffix(".csv") + gdf.to_csv(csv_filepath) + return csv_filepath + + +def seed_from_csv(track_csv_filepath, db_user, db_password, db_host, db_name): + file = Path(track_csv_filepath) + if not file.is_file(): + raise FileNotFoundError("No file exists at the location specified") + gdf = gpd.GeoDataFrame(pd.read_csv(file)) + gdf.fillna(np.nan).replace([np.nan], [None]) + gdf = gdf.drop(columns=["Unnamed: 0"]) + gdf.index.name = "id" + + engine = create_engine( + f"postgresql+psycopg2://{db_user}:{db_password}@{db_host}/{db_name}" + ) + + gdf.index += 1 # Making table 1-indexed + if inspect(engine).has_table("routemodelfsgp"): + row_len = pd.read_sql_query( + sql="SELECT COUNT(*) FROM routemodelfsgp", con=engine + ) + gdf.index += row_len.iloc[0, 0] + + response = gdf.to_sql( + name="routemodelfsgp", + con=engine, + schema="public", + if_exists="append", + index=True, + method="multi", + dtype={"geo": Geometry("POINT", srid=4326), "street_name": String}, + ) + + if gdf.shape[0] != response: + raise SystemError("dataframe insertion failed") + else: + print("routemodel dataframe insertion success") diff --git a/fsgp/requirements.txt b/fsgp/requirements.txt new file mode 100644 index 0000000..bba5168 Binary files /dev/null and b/fsgp/requirements.txt differ