diff --git a/seeker/report.txt b/seeker/report.txt index bd3e438f..2b844b34 100644 --- a/seeker/report.txt +++ b/seeker/report.txt @@ -1,3 +1,36 @@ +-------------------------------------------------------------------------------- + 2024-09-05 17:12:06.587562 +-------------------------------------------------------------------------------- + On branch main +Your branch is up to date with 'origin/main'. + +Changes not staged for commit: + (use "git add/rm ..." to update what will be committed) + (use "git restore ..." to discard changes in working directory) + deleted: snippet/ST_Point_table_from_csv.py + deleted: snippet/ascii-smuggler.py + deleted: snippet/bump + deleted: snippet/context_builder.py + deleted: snippet/dd.java + deleted: snippet/function.sh + deleted: snippet/instala_ccd_serpro_ubuntu.sh + deleted: snippet/login_to_google.py + deleted: snippet/polygon_table_from_datafile.py + deleted: snippet/sedona_df_to_geopandas.py + deleted: snippet/spatial_joins.py + +Untracked files: + (use "git add ..." to include in what will be committed) + snippet/IntroduceStaticSetter.java + snippet/IntroduceStaticSetter1.java + snippet/ReplaceGlobalReferenceWithGetter2.java + snippet/executor.py + snippet/mini_01_gugudan.java + snippet/snowpark_session.py + snippet/tap-linux.py + +no changes added to commit (use "git add" and/or "git commit -a") + -------------------------------------------------------------------------------- 2024-09-04 17:12:46.040880 -------------------------------------------------------------------------------- diff --git a/seeker/snippet/IntroduceStaticSetter.java b/seeker/snippet/IntroduceStaticSetter.java new file mode 100644 index 00000000..cd67f9f1 --- /dev/null +++ b/seeker/snippet/IntroduceStaticSetter.java @@ -0,0 +1,31 @@ +//date: 2024-09-05T16:52:22Z +//url: https://api.github.com/gists/b9c886d5f45c4da84e1e28746c104b79 +//owner: https://api.github.com/users/trikitrok + +class MessageRouter { + public void Route(Message message) { + //!! ouch... x( + ExternalRouter.getInstance().sendMessage(message); + } +} + +class ExternalRouter // another Singleton! x( +{ + private static ExternalRouter instance; + + private ExternalRouter() { + // initialize stuff + } + + public static ExternalRouter getInstance() { + if (instance == null) { + instance = new ExternalRouter(); + } + return instance; + } + + // more code... + public void sendMessage(Message message) { + // interesting code to send the message + } +} \ No newline at end of file diff --git a/seeker/snippet/IntroduceStaticSetter1.java b/seeker/snippet/IntroduceStaticSetter1.java new file mode 100644 index 00000000..3c4750fc --- /dev/null +++ b/seeker/snippet/IntroduceStaticSetter1.java @@ -0,0 +1,66 @@ +//date: 2024-09-05T17:06:13Z +//url: https://api.github.com/gists/6c259d3e1c1c3d3c0e02c580cbbdf1c5 +//owner: https://api.github.com/users/trikitrok + +class MessageRouter { + public void Route(Message message) { + //!! ouch... x( + ExternalRouter.getInstance().sendMessage(message); + } +} + +class ExternalRouter // another Singleton! x( +{ + private static ExternalRouter instance; + + private ExternalRouter() { + // initialize stuff + } + + public static ExternalRouter getInstance() { + if (instance == null) { + instance = new ExternalRouter(); + } + return instance; + } + + //!! Added for testing purposes only, do not use this in production code + public static void setInstanceForTesting(ExternalRouter anInstance) { + instance = anInstance; + } + + // more code... + public void sendMessage(Message message) { + // interesting code to send the message + } +} + +///////////////////////////////////////////// +// In some test we use the static setter to +// set a test double so that we can control +// what the singleton's instance does + +class MessageRouterTest +{ + @Test + public void routes_message() + { + ExternalRouter externalRouter = mock(ExternalRouter.class); + ExternalRouter.setInstanceForTesting(externalRouter); + MessageRouter messageRouter = new MessageRouter(); + Message message = new Message(); + + messageRouter.Route(message); + + verify(externalRouter).sendMessage(message); + } + + // some other tests... + + @AfterEach + public void TearDown() + { + // to keep tests isolated + ExternalRouter.setInstanceForTesting(null); + } +} \ No newline at end of file diff --git a/seeker/snippet/ReplaceGlobalReferenceWithGetter2.java b/seeker/snippet/ReplaceGlobalReferenceWithGetter2.java new file mode 100644 index 00000000..e99093f0 --- /dev/null +++ b/seeker/snippet/ReplaceGlobalReferenceWithGetter2.java @@ -0,0 +1,57 @@ +//date: 2024-09-05T16:46:15Z +//url: https://api.github.com/gists/278e3df2a2f14aff40d3162dd3cef94c +//owner: https://api.github.com/users/trikitrok + +// After applying Subclass & Override Method + +class RegisterSale { + private List items; + // more code... + + public void addItem(Barcode code) { + // using the Singleton!! x( + Item newItem = getInventory().getItemForBarCode(code); + items.add(newItem); + } + + protected Inventory getInventory() { + return Inventory.GetInstance(); + } + + // more code... +} + +///////////////////////////// + +// In some test + +public class RegisterSaleTest { + + @Test + public void Adds_An_Item() { + Barcode code = new Barcode(); + // some more setup code + //... + // we subclass & override the getter and return a test double of Inventory + Inventory inventory = mock(Inventory.class); + + when(inventory.getItemForBarCode(code)).thenReturn(AnItem().withBarcode(code).build()); + RegisterSaleForTesting registerSale = new RegisterSaleForTesting(inventory); + + // rest of the test... + } + + public class RegisterSaleForTesting extends RegisterSale { + private final Inventory inventory; + + public RegisterSaleForTesting(Inventory inventory) { + this.inventory = inventory; + } + + // overriden to separate from the singleton + @Override + protected Inventory getInventory() { + return inventory; + } + } +} \ No newline at end of file diff --git a/seeker/snippet/ST_Point_table_from_csv.py b/seeker/snippet/ST_Point_table_from_csv.py deleted file mode 100644 index 890f48e6..00000000 --- a/seeker/snippet/ST_Point_table_from_csv.py +++ /dev/null @@ -1,19 +0,0 @@ -#date: 2024-09-03T16:56:27Z -#url: https://api.github.com/gists/4e27bb8bae3bbff729866e1bc5866f8c -#owner: https://api.github.com/users/SeanLikesData - -point_csv_df = sedona.read.format("csv").\ - option("delimiter", ",").\ - option("header", "false").\ - load("data/testpoint.csv") - -point_csv_df.createOrReplaceTempView("pointtable") - -point_df = sedona.sql(" - select ST_Point( - cast(pointtable._c0 as Decimal(24,20)), - cast(pointtable._c1 as Decimal(24,20)) - ) - as arealandmark from pointtable - ") -point_df.show(5) \ No newline at end of file diff --git a/seeker/snippet/ascii-smuggler.py b/seeker/snippet/ascii-smuggler.py deleted file mode 100644 index 6e000d95..00000000 --- a/seeker/snippet/ascii-smuggler.py +++ /dev/null @@ -1,22 +0,0 @@ -#date: 2024-09-03T16:59:13Z -#url: https://api.github.com/gists/00507d3271b22c698a83e99990560bf8 -#owner: https://api.github.com/users/rossja - -# smuggle text hidden as invisible unicode chars -# credit to jthacker: https://x.com/rez0__/status/1745545813512663203 -# and embrace the red: https://embracethered.com/blog/posts/2024/hiding-and-finding-text-with-unicode-tags/ - -import pyperclip - -def convert_to_tag_chars(input_string): - return ''.join(chr(0xE0000 + ord(ch)) for ch in input_string) - -# Example usage: - -user_input = input("Enter a string to convert to tag characters: ") -tagged_output = convert_to_tag_chars(user_input) - -print(f"Tagged output:\n") -print(f"START{tagged_output}END") - -pyperclip.copy(tagged_output) \ No newline at end of file diff --git a/seeker/snippet/bump b/seeker/snippet/bump deleted file mode 100644 index d87b1cfe..00000000 --- a/seeker/snippet/bump +++ /dev/null @@ -1,27 +0,0 @@ -#date: 2024-09-03T17:12:01Z -#url: https://api.github.com/gists/2c8108dfaafccc0553fef195b88efe70 -#owner: https://api.github.com/users/garrettdimon - -#!/usr/bin/env bash - -# Exit on error -set -o errexit - -# Get the version from the .ruby-version file -target_ruby_version=$(cat .ruby-version) - -# Make sure the latest Ruby versions are present and available -arch -arm64 brew update && arch -arm64 brew upgrade ruby-build - -# List latest stable versions for each Ruby for verification -rbenv install --list - -# Install the version specified in the .ruby-version file -rbenv install $target_ruby_version - -# Wipe out the Gemfile.lock file -rm Gemfile.lock - -# Re-bundle and ensure linux is added -bundle install -bundle lock --add-platform x86_64-linux diff --git a/seeker/snippet/context_builder.py b/seeker/snippet/context_builder.py deleted file mode 100644 index 49a74ea3..00000000 --- a/seeker/snippet/context_builder.py +++ /dev/null @@ -1,12 +0,0 @@ -#date: 2024-09-03T16:56:27Z -#url: https://api.github.com/gists/4e27bb8bae3bbff729866e1bc5866f8c -#owner: https://api.github.com/users/SeanLikesData - -config = SedonaContext.builder() .\ - config('spark.jars.packages', - 'org.apache.sedona:sedona-spark-3.4_2.12:1.6.0,' - 'org.datasyslab:geotools-wrapper:1.6.0-28.2,' - 'uk.co.gresearch.spark:spark-extension_2.12:2.11.0-3.4'). \ - getOrCreate() - -sedona = SedonaContext.create(config) \ No newline at end of file diff --git a/seeker/snippet/dd.java b/seeker/snippet/dd.java deleted file mode 100644 index 140965ac..00000000 --- a/seeker/snippet/dd.java +++ /dev/null @@ -1,60 +0,0 @@ -//date: 2024-09-03T17:01:38Z -//url: https://api.github.com/gists/28083a29785c2f6a3ec5816a8320a739 -//owner: https://api.github.com/users/jorgecuza92 - -package com.capitalone.engagementexperience.reststepdefinitions.directdeposit; - -import com.capitalone.banxoa.cucumber.AbstractRESTStepDefinition; -import com.capitalone.engagementexperience.model.ScenarioData; -import org.springframework.beans.factory.annotation.Autowired; - -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.concurrent.TimeUnit; - -public class DirectDepositStepDefinitions extends AbstractRESTStepDefinition { - - // Autowired to inject ScenarioData, which contains transaction data - @Autowired - private ScenarioData scenarioData; - - // Method to set the enrollment status of Direct Deposit based on input and transaction time - @And("^the account IsDirectDepositEnrolled value is (string)$") - public void setScenarioDataToSupportDirectDepositEnrolment(String isDirectDepositEnrolledString) throws Exception { - // Parse the input string to determine if direct deposit is enrolled - boolean isDirectDepositEnrolled = Boolean.parseBoolean(isDirectDepositEnrolledString); - - // Fetch the last transaction time; replace with your method to get the correct date string if necessary - String lastTransactionTime = scenarioData.getLastTransactionTime(); // Ensure this fetches the correct timestamp - - // Convert the lastTransactionTime from String to Date object - DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); // Adjust format if necessary - Date transactionDate = dateFormat.parse(lastTransactionTime); - Date currentDate = new Date(); // Get the current date and time - - // Calculate the difference in days between the current date and the last transaction date - long differenceInMillis = currentDate.getTime() - transactionDate.getTime(); - long daysDifference = TimeUnit.MILLISECONDS.toDays(differenceInMillis); - - // Logic to determine if the enrollment status needs to be updated - if (isDirectDepositEnrolled && daysDifference > 15) { - // Update status to unenrolled if last transaction is older than 15 days - scenarioData.setClientCorrelationId("unenrolled"); - scenarioData.setCustomerReferenceId("Direct Deposit inactive for over 15 days"); - System.out.println("Updated: Direct Deposit is not currently enrolled due to inactivity."); - } else { - // Keep enrollment as active if within 15 days - scenarioData.setClientCorrelationId("enrolled"); - scenarioData.setCustomerReferenceId("Direct Deposit active"); - } - } - - // Example of another existing method for reference, no changes needed - @And("^invalid (string) is provided in scenario$") - public void setInvalidFieldInScenarioData(String fieldName) { - if (!fieldName.equals("CustomerReferenceId")) { - scenarioData.setClientCorrelationId("invalid"); - } - } -} \ No newline at end of file diff --git a/seeker/snippet/executor.py b/seeker/snippet/executor.py new file mode 100644 index 00000000..8f0dd85f --- /dev/null +++ b/seeker/snippet/executor.py @@ -0,0 +1,987 @@ +#date: 2024-09-05T16:52:32Z +#url: https://api.github.com/gists/0dd67670f738f1dc2bc24a885ab78914 +#owner: https://api.github.com/users/seeker25 + +from __future__ import annotations + +import contextlib +import csv +import functools +import itertools +import json +import threading + +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import wait +from pathlib import Path +from subprocess import CalledProcessError +from typing import TYPE_CHECKING +from typing import Any + +from cleo.io.null_io import NullIO +from poetry.core.packages.utils.link import Link + +from poetry.installation.chef import Chef +from poetry.installation.chef import ChefBuildError +from poetry.installation.chef import ChefInstallError +from poetry.installation.chooser import Chooser +from poetry.installation.operations import Install +from poetry.installation.operations import Uninstall +from poetry.installation.operations import Update +from poetry.installation.wheel_installer import WheelInstaller +from poetry.puzzle.exceptions import SolverProblemError +from poetry.utils._compat import decode +from poetry.utils.authenticator import Authenticator +from poetry.utils.env import EnvCommandError +from poetry.utils.helpers import Downloader +from poetry.utils.helpers import get_file_hash +from poetry.utils.helpers import get_highest_priority_hash_type +from poetry.utils.helpers import pluralize +from poetry.utils.helpers import remove_directory +from poetry.utils.pip import pip_install + + +if TYPE_CHECKING: + from cleo.io.io import IO + from cleo.io.outputs.section_output import SectionOutput + from poetry.core.masonry.builders.builder import Builder + from poetry.core.packages.package import Package + + from poetry.config.config import Config + from poetry.installation.operations.operation import Operation + from poetry.repositories import RepositoryPool + from poetry.utils.env import Env + + +class Executor: + def __init__( + self, + env: Env, + pool: RepositoryPool, + config: Config, + io: IO, + parallel: bool | None = None, + disable_cache: bool = False, + ) -> None: + self._env = env + self._io = io + self._dry_run = False + self._enabled = True + self._verbose = False + self._wheel_installer = WheelInstaller(self._env) + self._use_modern_installation = config.get( + "installer.modern-installation", True + ) + if not self._use_modern_installation: + self._io.write_line( + "Warning: Setting `installer.modern-installation` to `false` " + "is deprecated." + ) + self._io.write_line( + "The pip-based installer will be removed in a future release." + ) + self._io.write_line( + "See https://github.com/python-poetry/poetry/issues/8987." + ) + + if parallel is None: + parallel = config.get("installer.parallel", True) + + if parallel: + self._max_workers = config.installer_max_workers + else: + self._max_workers = 1 + + self._artifact_cache = pool.artifact_cache + self._authenticator = Authenticator( + config, self._io, disable_cache=disable_cache, pool_size=self._max_workers + ) + self._chef = Chef(self._artifact_cache, self._env, pool) + self._chooser = Chooser(pool, self._env, config) + + self._executor = ThreadPoolExecutor(max_workers=self._max_workers) + self._total_operations = 0 + self._executed_operations = 0 + self._executed = {"install": 0, "update": 0, "uninstall": 0} + self._skipped = {"install": 0, "update": 0, "uninstall": 0} + self._sections: dict[int, SectionOutput] = {} + self._yanked_warnings: list[str] = [] + self._lock = threading.Lock() + self._shutdown = False + self._hashes: dict[str, str] = {} + + # Cache whether decorated output is supported. + # https://github.com/python-poetry/cleo/issues/423 + self._decorated_output: bool = self._io.output.is_decorated() + + @property + def installations_count(self) -> int: + return self._executed["install"] + + @property + def updates_count(self) -> int: + return self._executed["update"] + + @property + def removals_count(self) -> int: + return self._executed["uninstall"] + + @property + def enabled(self) -> bool: + return self._enabled + + def supports_fancy_output(self) -> bool: + return self._decorated_output and not self._dry_run + + def disable(self) -> Executor: + self._enabled = False + + return self + + def dry_run(self, dry_run: bool = True) -> Executor: + self._dry_run = dry_run + + return self + + def verbose(self, verbose: bool = True) -> Executor: + self._verbose = verbose + + return self + + def enable_bytecode_compilation(self, enable: bool = True) -> None: + self._wheel_installer.enable_bytecode_compilation(enable) + + def pip_install( + self, req: Path, upgrade: bool = False, editable: bool = False + ) -> int: + try: + pip_install(req, self._env, upgrade=upgrade, editable=editable) + except EnvCommandError as e: + output = decode(e.e.output) + if ( + "KeyboardInterrupt" in output + or "ERROR: Operation cancelled by user" in output + ): + return -2 + raise + + return 0 + + def execute(self, operations: list[Operation]) -> int: + self._total_operations = len(operations) + for job_type in self._executed: + self._executed[job_type] = 0 + self._skipped[job_type] = 0 + + if operations and (self._enabled or self._dry_run): + self._display_summary(operations) + + self._sections = {} + self._yanked_warnings = [] + + # pip has to be installed first without parallelism if we install via pip + for i, op in enumerate(operations): + if op.package.name == "pip": + wait([self._executor.submit(self._execute_operation, op)]) + del operations[i] + break + + git_repositories_to_clone = set() + # We group operations by priority + groups = itertools.groupby(operations, key=lambda o: -o.priority) + for _, group in groups: + tasks = [] + serial_operations = [] + for operation in group: + if self._shutdown: + break + + # It's not safe to clone the same git repository multiple times in parallel + if operation.package.source_type == "git" and not operation.skipped: + # If this is the first operation cloning this repository, + # allow it to be executed in parallel + if operation.package.source_url not in git_repositories_to_clone: + git_repositories_to_clone.add(operation.package.source_url) + + # If another operation is already cloning this repository, + # it must be executed serially + else: + serial_operations.append(operation) + continue + + # Some operations are unsafe, we must execute them serially in a group + # https://github.com/python-poetry/poetry/issues/3086 + # https://github.com/python-poetry/poetry/issues/2658 + # + # We need to explicitly check source type here, see: + # https://github.com/python-poetry/poetry-core/pull/98 + is_parallel_unsafe = operation.job_type == "uninstall" or ( + operation.package.develop + and operation.package.source_type in {"directory", "git"} + ) + if not operation.skipped and is_parallel_unsafe: + serial_operations.append(operation) + continue + + tasks.append(self._executor.submit(self._execute_operation, operation)) + + try: + wait(tasks) + + for operation in serial_operations: + wait([self._executor.submit(self._execute_operation, operation)]) + + except KeyboardInterrupt: + self._shutdown = True + + if self._shutdown: + # Cancelling further tasks from being executed + [task.cancel() for task in tasks] + self._executor.shutdown(wait=True) + + break + + for warning in self._yanked_warnings: + self._io.write_error_line(f"Warning: {warning}") + for path, issues in self._wheel_installer.invalid_wheels.items(): + formatted_issues = "\n".join(issues) + warning = ( + f"Validation of the RECORD file of {path.name} failed." + " Please report to the maintainers of that package so they can fix" + f" their build process. Details:\n{formatted_issues}\n" + ) + self._io.write_error_line(f"Warning: {warning}") + + return 1 if self._shutdown else 0 + + def _write(self, operation: Operation, line: str) -> None: + if not self.supports_fancy_output() or not self._should_write_operation( + operation + ): + return + + if self._io.is_debug(): + with self._lock: + section = self._sections[id(operation)] + section.write_line(line) + + return + + with self._lock: + section = self._sections[id(operation)] + section.clear() + section.write(line) + + def _execute_operation(self, operation: Operation) -> None: + try: + op_message = self.get_operation_message(operation) + if self.supports_fancy_output(): + if id(operation) not in self._sections and self._should_write_operation( + operation + ): + with self._lock: + self._sections[id(operation)] = self._io.section() + self._sections[id(operation)].write_line( + f" - {op_message}:" + " Pending..." + ) + else: + if self._should_write_operation(operation): + if not operation.skipped: + self._io.write_line( + f" - {op_message}" + ) + else: + self._io.write_line( + f" - {op_message}: " + "Skipped " + "for the following reason: " + f"{operation.skip_reason}" + ) + + try: + result = self._do_execute_operation(operation) + except EnvCommandError as e: + if e.e.returncode == -2: + result = -2 + else: + raise + + # If we have a result of -2 it means a KeyboardInterrupt + # in the any python subprocess, so we raise a KeyboardInterrupt + # error to be picked up by the error handler. + if result == -2: + raise KeyboardInterrupt + except Exception as e: + try: + from cleo.ui.exception_trace import ExceptionTrace + + io: IO | SectionOutput + if not self.supports_fancy_output(): + io = self._io + else: + message = ( + " -" + f" {self.get_operation_message(operation, error=True)}:" + " Failed" + ) + self._write(operation, message) + io = self._sections.get(id(operation), self._io) + + with self._lock: + trace = ExceptionTrace(e) + trace.render(io) + pkg = operation.package + if isinstance(e, ChefBuildError): + pip_command = "pip wheel --no-cache-dir --use-pep517" + if pkg.develop: + requirement = pkg.source_url + pip_command += " --editable" + else: + requirement = ( + pkg.to_dependency().to_pep_508().split(";")[0].strip() + ) + message = ( + "" + "Note: This error originates from the build backend," + " and is likely not a problem with poetry" + f" but with {pkg.pretty_name} ({pkg.full_pretty_version})" + " not supporting PEP 517 builds. You can verify this by" + f" running '{pip_command} \"{requirement}\"'." + "" + ) + elif isinstance(e, ChefInstallError): + message = ( + "" + "Cannot install build-system.requires" + f" for {pkg.pretty_name}." + "" + ) + elif isinstance(e, SolverProblemError): + message = ( + "" + "Cannot resolve build-system.requires" + f" for {pkg.pretty_name}." + "" + ) + else: + message = f"Cannot install {pkg.pretty_name}." + + io.write_line("") + io.write_line(message) + io.write_line("") + finally: + with self._lock: + self._shutdown = True + + except KeyboardInterrupt: + try: + message = ( + " -" + f" {self.get_operation_message(operation, warning=True)}:" + " Cancelled" + ) + if not self.supports_fancy_output(): + self._io.write_line(message) + else: + self._write(operation, message) + finally: + with self._lock: + self._shutdown = True + + def _do_execute_operation(self, operation: Operation) -> int: + method = operation.job_type + + operation_message = self.get_operation_message(operation) + if operation.skipped: + if self.supports_fancy_output(): + self._write( + operation, + f" - {operation_message}: " + "Skipped " + "for the following reason: " + f"{operation.skip_reason}", + ) + + self._skipped[operation.job_type] += 1 + + return 0 + + if not self._enabled or self._dry_run: + return 0 + + result: int = getattr(self, f"_execute_{method}")(operation) + + if result != 0: + return result + + operation_message = self.get_operation_message(operation, done=True) + message = f" - {operation_message}" + self._write(operation, message) + + self._increment_operations_count(operation, True) + + return result + + def _increment_operations_count(self, operation: Operation, executed: bool) -> None: + with self._lock: + if executed: + self._executed_operations += 1 + self._executed[operation.job_type] += 1 + else: + self._skipped[operation.job_type] += 1 + + def run_pip(self, *args: Any, **kwargs: Any) -> int: + try: + self._env.run_pip(*args, **kwargs) + except EnvCommandError as e: + output = decode(e.e.output) + if ( + "KeyboardInterrupt" in output + or "ERROR: Operation cancelled by user" in output + ): + return -2 + + raise + + return 0 + + def get_operation_message( + self, + operation: Operation, + done: bool = False, + error: bool = False, + warning: bool = False, + ) -> str: + base_tag = "fg=default" + operation_color = "c2" + source_operation_color = "c2" + package_color = "c1" + + if error: + operation_color = "error" + elif warning: + operation_color = "warning" + elif done: + operation_color = "success" + + if operation.skipped: + base_tag = "fg=default;options=dark" + operation_color += "_dark" + source_operation_color += "_dark" + package_color += "_dark" + + if isinstance(operation, Install): + return ( + f"<{base_tag}>Installing" + f" <{package_color}>{operation.package.name}" + f" (<{operation_color}>{operation.package.full_pretty_version})" + ) + + if isinstance(operation, Uninstall): + return ( + f"<{base_tag}>Removing" + f" <{package_color}>{operation.package.name}" + f" (<{operation_color}>{operation.package.full_pretty_version})" + ) + + if isinstance(operation, Update): + initial_version = (initial_pkg := operation.initial_package).version + target_version = (target_pkg := operation.target_package).version + update_kind = ( + "Updating" if target_version >= initial_version else "Downgrading" + ) + return ( + f"<{base_tag}>{update_kind}" + f" <{package_color}>{initial_pkg.name} " + f"(<{source_operation_color}>" + f"{initial_pkg.full_pretty_version}" + f" -> <{operation_color}>" + f"{target_pkg.full_pretty_version})" + ) + return "" + + def _display_summary(self, operations: list[Operation]) -> None: + installs = 0 + updates = 0 + uninstalls = 0 + skipped = 0 + for op in operations: + if op.skipped: + skipped += 1 + continue + + if op.job_type == "install": + installs += 1 + elif op.job_type == "update": + updates += 1 + elif op.job_type == "uninstall": + uninstalls += 1 + + if not installs and not updates and not uninstalls and not self._verbose: + self._io.write_line("") + self._io.write_line("No dependencies to install or update") + + return + + self._io.write_line("") + self._io.write("Package operations: ") + self._io.write(f"{installs} install{pluralize(installs)}, ") + self._io.write(f"{updates} update{pluralize(updates)}, ") + self._io.write(f"{uninstalls} removal{pluralize(uninstalls)}") + if skipped and self._verbose: + self._io.write(f", {skipped} skipped") + self._io.write_line("") + self._io.write_line("") + + def _execute_install(self, operation: Install | Update) -> int: + status_code = self._install(operation) + + self._save_url_reference(operation) + + return status_code + + def _execute_update(self, operation: Install | Update) -> int: + status_code = self._update(operation) + + self._save_url_reference(operation) + + return status_code + + def _execute_uninstall(self, operation: Uninstall) -> int: + op_msg = self.get_operation_message(operation) + message = f" - {op_msg}: Removing..." + self._write(operation, message) + + return self._remove(operation.package) + + def _install(self, operation: Install | Update) -> int: + package = operation.package + if package.source_type == "directory" and not self._use_modern_installation: + return self._install_directory_without_wheel_installer(operation) + + cleanup_archive: bool = False + if package.source_type == "git": + archive = self._prepare_git_archive(operation) + cleanup_archive = operation.package.develop + elif package.source_type == "file": + archive = self._prepare_archive(operation) + elif package.source_type == "directory": + archive = self._prepare_archive(operation) + cleanup_archive = True + elif package.source_type == "url": + assert package.source_url is not None + archive = self._download_link(operation, Link(package.source_url)) + else: + archive = self._download(operation) + + operation_message = self.get_operation_message(operation) + message = ( + f" - {operation_message}:" + " Installing..." + ) + self._write(operation, message) + + if not self._use_modern_installation: + return self.pip_install(archive, upgrade=operation.job_type == "update") + + try: + if operation.job_type == "update": + # Uninstall first + # TODO: Make an uninstaller and find a way to rollback in case + # the new package can't be installed + assert isinstance(operation, Update) + self._remove(operation.initial_package) + + self._wheel_installer.install(archive) + finally: + if cleanup_archive: + archive.unlink() + + return 0 + + def _update(self, operation: Install | Update) -> int: + return self._install(operation) + + def _remove(self, package: Package) -> int: + # If we have a VCS package, remove its source directory + if package.source_type == "git": + src_dir = self._env.path / "src" / package.name + if src_dir.exists(): + remove_directory(src_dir, force=True) + + try: + return self.run_pip("uninstall", package.name, "-y") + except CalledProcessError as e: + if "not installed" in str(e): + return 0 + + raise + + def _prepare_archive( + self, operation: Install | Update, *, output_dir: Path | None = None + ) -> Path: + package = operation.package + operation_message = self.get_operation_message(operation) + + message = ( + f" - {operation_message}:" + " Preparing..." + ) + self._write(operation, message) + + assert package.source_url is not None + archive = Path(package.source_url) + if package.source_subdirectory: + archive = archive / package.source_subdirectory + if not Path(package.source_url).is_absolute() and package.root_dir: + archive = package.root_dir / archive + + self._populate_hashes_dict(archive, package) + + return self._chef.prepare( + archive, editable=package.develop, output_dir=output_dir + ) + + def _prepare_git_archive(self, operation: Install | Update) -> Path: + from poetry.vcs.git import Git + + package = operation.package + assert package.source_url is not None + + if package.source_resolved_reference and not package.develop: + # Only cache git archives when we know precise reference hash, + # otherwise we might get stale archives + cached_archive = self._artifact_cache.get_cached_archive_for_git( + package.source_url, + package.source_resolved_reference, + package.source_subdirectory, + env=self._env, + ) + if cached_archive is not None: + return cached_archive + + operation_message = self.get_operation_message(operation) + + message = ( + f" - {operation_message}: Cloning..." + ) + self._write(operation, message) + + source = Git.clone( + url=package.source_url, + source_root=self._env.path / "src", + revision=package.source_resolved_reference or package.source_reference, + ) + + # Now we just need to install from the source directory + original_url = package.source_url + package._source_url = str(source.path) + + output_dir = None + if package.source_resolved_reference and not package.develop: + output_dir = self._artifact_cache.get_cache_directory_for_git( + original_url, + package.source_resolved_reference, + package.source_subdirectory, + ) + + archive = self._prepare_archive(operation, output_dir=output_dir) + if not package.develop: + package._source_url = original_url + + if output_dir is not None and output_dir.is_dir(): + # Mark directories with cached git packages, to distinguish from + # "normal" cache + (output_dir / ".created_from_git_dependency").touch() + + return archive + + def _install_directory_without_wheel_installer( + self, operation: Install | Update + ) -> int: + from poetry.factory import Factory + from poetry.pyproject.toml import PyProjectTOML + + package = operation.package + operation_message = self.get_operation_message(operation) + + message = ( + f" - {operation_message}:" + " Building..." + ) + self._write(operation, message) + + assert package.source_url is not None + if package.root_dir: + req = package.root_dir / package.source_url + else: + req = Path(package.source_url).resolve(strict=False) + + if package.source_subdirectory: + req /= package.source_subdirectory + + pyproject = PyProjectTOML(req / "pyproject.toml") + + package_poetry = None + if pyproject.is_poetry_project(): + with contextlib.suppress(RuntimeError): + package_poetry = Factory().create_poetry(pyproject.file.path.parent) + + if package_poetry is not None: + builder: Builder + if package.develop and not package_poetry.package.build_script: + from poetry.masonry.builders.editable import EditableBuilder + + # This is a Poetry package in editable mode + # we can use the EditableBuilder without going through pip + # to install it, unless it has a build script. + builder = EditableBuilder(package_poetry, self._env, NullIO()) + builder.build() + + return 0 + + if package_poetry.package.build_script: + from poetry.core.masonry.builders.sdist import SdistBuilder + + builder = SdistBuilder(package_poetry) + with builder.setup_py(): + return self.pip_install(req, upgrade=True, editable=package.develop) + + return self.pip_install(req, upgrade=True, editable=package.develop) + + def _download(self, operation: Install | Update) -> Path: + link = self._chooser.choose_for(operation.package) + + if link.yanked: + # Store yanked warnings in a list and print after installing, so they can't + # be overlooked. Further, printing them in the concerning section would have + # the risk of overwriting the warning, so it is only briefly visible. + message = ( + f"The file chosen for install of {operation.package.pretty_name} " + f"{operation.package.pretty_version} ({link.show_url}) is yanked." + ) + if link.yanked_reason: + message += f" Reason for being yanked: {link.yanked_reason}" + self._yanked_warnings.append(message) + + return self._download_link(operation, link) + + def _download_link(self, operation: Install | Update, link: Link) -> Path: + package = operation.package + + # Get original package for the link provided + download_func = functools.partial(self._download_archive, operation) + original_archive = self._artifact_cache.get_cached_archive_for_link( + link, strict=True, download_func=download_func + ) + + # Get potential higher prioritized cached archive, otherwise it will fall back + # to the original archive. + archive = self._artifact_cache.get_cached_archive_for_link( + link, + strict=False, + env=self._env, + ) + if archive is None: + # Since we previously downloaded an archive, we now should have + # something cached that we can use here. The only case in which + # archive is None is if the original archive is not valid for the + # current environment. + raise RuntimeError( + f"Package {link.url} cannot be installed in the current environment" + f" {self._env.marker_env}" + ) + + if archive.suffix != ".whl": + message = ( + f" - {self.get_operation_message(operation)}:" + " Preparing..." + ) + self._write(operation, message) + + archive = self._chef.prepare(archive, output_dir=original_archive.parent) + + # Use the original archive to provide the correct hash. + self._populate_hashes_dict(original_archive, package) + + return archive + + def _populate_hashes_dict(self, archive: Path, package: Package) -> None: + if package.files and archive.name in {f["file"] for f in package.files}: + archive_hash = self._validate_archive_hash(archive, package) + self._hashes[package.name] = archive_hash + + @staticmethod + def _validate_archive_hash(archive: Path, package: Package) -> str: + known_hashes = {f["hash"] for f in package.files if f["file"] == archive.name} + hash_types = {t.split(":")[0] for t in known_hashes} + hash_type = get_highest_priority_hash_type(hash_types, archive.name) + + if hash_type is None: + raise RuntimeError( + f"No usable hash type(s) for {package} from archive" + f" {archive.name} found (known hashes: {known_hashes!s})" + ) + + archive_hash = f"{hash_type}:{get_file_hash(archive, hash_type)}" + + if archive_hash not in known_hashes: + raise RuntimeError( + f"Hash for {package} from archive {archive.name} not found in" + f" known hashes (was: {archive_hash})" + ) + + return archive_hash + + def _download_archive( + self, + operation: Install | Update, + url: str, + dest: Path, + ) -> None: + downloader = Downloader(url, dest, self._authenticator) + wheel_size = downloader.total_size + + operation_message = self.get_operation_message(operation) + message = ( + f" - {operation_message}: Downloading..." + ) + progress = None + if self.supports_fancy_output(): + if wheel_size is None: + self._write(operation, message) + else: + from cleo.ui.progress_bar import ProgressBar + + progress = ProgressBar( + self._sections[id(operation)], max=int(wheel_size) + ) + progress.set_format(message + " %percent%%") + + if progress: + with self._lock: + self._sections[id(operation)].clear() + progress.start() + + for fetched_size in downloader.download_with_progress(chunk_size=4096): + if progress: + with self._lock: + progress.set_progress(fetched_size) + + if progress: + with self._lock: + progress.finish() + + def _should_write_operation(self, operation: Operation) -> bool: + return ( + not operation.skipped or self._dry_run or self._verbose or not self._enabled + ) + + def _save_url_reference(self, operation: Operation) -> None: + """ + Create and store a PEP-610 `direct_url.json` file, if needed. + """ + if operation.job_type not in {"install", "update"}: + return + + package = operation.package + + if not package.source_url or package.source_type == "legacy": + if not self._use_modern_installation: + # Since we are installing from our own distribution cache pip will write + # a `direct_url.json` file pointing to the cache distribution. + # + # That's not what we want, so we remove the direct_url.json file, if it + # exists. + for ( + direct_url_json + ) in self._env.site_packages.find_distribution_direct_url_json_files( + distribution_name=package.name, writable_only=True + ): + direct_url_json.unlink(missing_ok=True) + return + + url_reference: dict[str, Any] | None = None + + if package.source_type == "git" and not package.develop: + url_reference = self._create_git_url_reference(package) + elif package.source_type in ("directory", "git"): + url_reference = self._create_directory_url_reference(package) + elif package.source_type == "url": + url_reference = self._create_url_url_reference(package) + elif package.source_type == "file": + url_reference = self._create_file_url_reference(package) + + if url_reference: + for dist in self._env.site_packages.distributions( + name=package.name, writable_only=True + ): + dist_path = dist._path # type: ignore[attr-defined] + assert isinstance(dist_path, Path) + url = dist_path / "direct_url.json" + url.write_text(json.dumps(url_reference), encoding="utf-8") + + record = dist_path / "RECORD" + if record.exists(): + with record.open(mode="a", encoding="utf-8", newline="") as f: + writer = csv.writer(f) + path = url.relative_to(record.parent.parent) + writer.writerow([str(path), "", ""]) + + def _create_git_url_reference(self, package: Package) -> dict[str, Any]: + reference = { + "url": package.source_url, + "vcs_info": { + "vcs": "git", + "requested_revision": package.source_reference, + "commit_id": package.source_resolved_reference, + }, + } + if package.source_subdirectory: + reference["subdirectory"] = package.source_subdirectory + + return reference + + def _create_url_url_reference(self, package: Package) -> dict[str, Any]: + archive_info = self._get_archive_info(package) + + return {"url": package.source_url, "archive_info": archive_info} + + def _create_file_url_reference(self, package: Package) -> dict[str, Any]: + archive_info = self._get_archive_info(package) + + assert package.source_url is not None + return { + "url": Path(package.source_url).as_uri(), + "archive_info": archive_info, + } + + def _create_directory_url_reference(self, package: Package) -> dict[str, Any]: + dir_info = {} + + if package.develop: + dir_info["editable"] = True + + assert package.source_url is not None + return { + "url": Path(package.source_url).as_uri(), + "dir_info": dir_info, + } + + def _get_archive_info(self, package: Package) -> dict[str, Any]: + """ + Create dictionary `archive_info` for file `direct_url.json`. + + Specification: https://packaging.python.org/en/latest/specifications/direct-url + (it supersedes PEP 610) + + :param package: This must be a poetry package instance. + """ + archive_info = {} + + if package.name in self._hashes: + algorithm, value = self._hashes[package.name].split(":") + archive_info["hashes"] = {algorithm: value} + + return archive_info \ No newline at end of file diff --git a/seeker/snippet/function.sh b/seeker/snippet/function.sh deleted file mode 100644 index e90f81e2..00000000 --- a/seeker/snippet/function.sh +++ /dev/null @@ -1,30 +0,0 @@ -#date: 2024-09-03T16:48:38Z -#url: https://api.github.com/gists/efda108268ac08bb45984f8bacf965aa -#owner: https://api.github.com/users/mkpanq - -# Function to set the latest version of a package locally and install if necessary -asdf_latest_local() { - local plugin="$1" - local latest_version - - # Check if a plugin name is provided - if [[ -z "$plugin" ]]; then - echo "Usage: asdf_latest_local " - return 1 - fi - - # Get the latest version available for the plugin - latest_version=$(asdf latest "$plugin") - - # Check if the latest version is already installed - if ! asdf list "$plugin" | grep -q "$latest_version"; then - echo "Installing latest version of $plugin: $latest_version" - asdf install "$plugin" "$latest_version" - else - echo "Latest version of $plugin is already installed: $latest_version" - fi - - # Set the latest version locally - echo "Setting $plugin to use version $latest_version locally" - asdf local "$plugin" "$latest_version" -} diff --git a/seeker/snippet/instala_ccd_serpro_ubuntu.sh b/seeker/snippet/instala_ccd_serpro_ubuntu.sh deleted file mode 100644 index 2d956303..00000000 --- a/seeker/snippet/instala_ccd_serpro_ubuntu.sh +++ /dev/null @@ -1,40 +0,0 @@ -#date: 2024-09-03T16:48:17Z -#url: https://api.github.com/gists/2b409571e3939324a53e30c8ffad8b5d -#owner: https://api.github.com/users/rodrigomaia - -# Entrar na pasta de certificados do sistema: -# Ubuntu anterior ao 18: -sudo mkdir /usr/share/ca-certificates/serpro/ -sudo cd /usr/share/ca-certificates/serpro/ -# Ubuntu 18: -sudo mkdir /usr/local/share/ca-certificates/serpro/ -cd /usr/local/share/ca-certificates/serpro/ - -# Baixar os certificados do repositorio: -wget -r --no-check-certificate https://ccd.serpro.gov.br/serproacf/docs/ - -# Remover apenas os certificados de interesse: -find ./ccd.serpro.gov.br -name *.crt | sudo xargs -I{} cp -u {} . - -# Limpar o restante do wget: -sudo rm -rf ccd.serpro.gov.br/ - -# Executar compilação dos certificados para o sistema: -sudo update-ca-certificates - -# Instalar os certificados nos browsers -sudo apt install libnss3-tools - -# Instalar os certificados no google-chrome -for i in $(ls /usr/local/share/ca-certificates/serpro/); do $(certutil -d sql:$HOME/.pki/nssdb -A -t "C,C,C" -n $i -i /usr/local/share/ca-certificates/serpro/$i); done - -# Instalar os certificados no firefox -cat ~/.mozilla/firefox/profiles.ini - -# Anote o valor do Default no meu caso: yxsfy966.default-release -# [Install4F96D1932A9F858E] -# Default=yxsfy966.default-release -# Locked=1 - -# O Resultado você subistitui no comando abaixo logo depois do $HOME/.mozilla/firefox/: -for i in $(ls /usr/local/share/ca-certificates/serpro/); do $(certutil -d sql:$HOME/.mozilla/firefox/yxsfy966.default-release -A -t "C,C,C" -n $i -i /usr/local/share/ca-certificates/serpro/$i); done \ No newline at end of file diff --git a/seeker/snippet/login_to_google.py b/seeker/snippet/login_to_google.py deleted file mode 100644 index f62acdee..00000000 --- a/seeker/snippet/login_to_google.py +++ /dev/null @@ -1,97 +0,0 @@ -#date: 2024-09-03T17:08:18Z -#url: https://api.github.com/gists/7e30a1630f9c2d59d619b4b7e4008e2e -#owner: https://api.github.com/users/samuelvillegas - -""" -The most important part is replacing the word `headless` from the user Agent -""" -import selenium.webdriver.chrome.webdriver -import selenium.webdriver.chrome.options -from selenium.webdriver.common.by import By -from selenium.webdriver.support.ui import WebDriverWait -from selenium.webdriver.support import expected_conditions as EC -from selenium.webdriver.common.keys import Keys -import time - -email = 'example@mail.com' -password = "**********" - -# Setting up Chrome options -driver_opts = selenium.webdriver.chrome.options.Options() -driver_opts.add_argument('--no-sandbox') -driver_opts.add_argument('--disable-dev-shm-usage') - -# Disable GPU hardware acceleration -# To prevent gpu related issues -driver_opts.add_argument('--disable-gpu') - -driver_opts.add_argument('--window-size=1920,1080') -driver_opts.add_argument('--start-maximized') - -# Disable the Blink feature that detects automation -driver_opts.add_argument('--disable-blink-features=AutomationControlled') - -# Exclude the 'enable-automation' switch to make automation less detectable -driver_opts.add_experimental_option('excludeSwitches', ['enable-automation']) - -# Disable the use of the automation extension -driver_opts.add_experimental_option('useAutomationExtension', False) - -# driver_opts.add_argument('--incognito') # Open the browser in incognito mode - -# Run the browser in headless mode -driver_opts.add_argument('--headless') - -# Initialize Chrome WebDriver -driver = selenium.webdriver.chrome.webdriver.WebDriver(options=driver_opts) - -# Remove 'Headless' from the user agent -# So, websites can't detect that the browser is running in headless mode -current_user_agent = driver.execute_script("return navigator.userAgent;") -driver.execute_cdp_cmd( - 'Network.setUserAgentOverride', - { - "userAgent": current_user_agent.replace('Headless', ''), - }, -) - -# Prevent websites from detecting that the browser is being controlled by automation tools -driver.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', { - 'source': ''' - Object.defineProperty(navigator, 'webdriver', { - get: () => undefined - }); - ''' -}) - -try: - # Open Google login page - driver.get('https://accounts.google.com/signin') - - # Enter email or phone - email_input = WebDriverWait(driver, 10).until( - EC.presence_of_element_located((By.ID, 'identifierId')) - ) - time.sleep(2) # Small delay - email_input.send_keys(email) - email_input.send_keys(Keys.RETURN) - - # Wait for password input to appear and enter password - password_input = "**********" - EC.presence_of_element_located((By.NAME, 'Passwd')) - ) - time.sleep(2) # Small delay - password_input.send_keys(password) - password_input.send_keys(Keys.RETURN) - - # Wait for successful login (Example: Google home page) - WebDriverWait(driver, 10).until( - EC.title_contains('Google'), - ) - print("Login successful!") - -finally: - # Close the browser - driver.quit() -ser - driver.quit() diff --git a/seeker/snippet/mini_01_gugudan.java b/seeker/snippet/mini_01_gugudan.java new file mode 100644 index 00000000..870e4dcd --- /dev/null +++ b/seeker/snippet/mini_01_gugudan.java @@ -0,0 +1,25 @@ +//date: 2024-09-05T17:03:58Z +//url: https://api.github.com/gists/a7e890c666bbee231c2f5169198373f3 +//owner: https://api.github.com/users/mu-nss + +/* + * 과제1. 콘솔 화면에 구구단 출력하기 + * 과제 제출자: 문소정 + */ + +public class Main { + public static void main(String[] args) { + + String result; + + System.out.println("[구구단 출력]"); + for (int row = 1; row < 10; row++) { + for (int col = 1; col < 10; col++) { + result = String.format("%02d x %02d = %02d\t", col, row, row * col); + System.out.print(result); + } + System.out.println(); + } + + } // end of main() +}// end of Main \ No newline at end of file diff --git a/seeker/snippet/polygon_table_from_datafile.py b/seeker/snippet/polygon_table_from_datafile.py deleted file mode 100644 index b942e5f8..00000000 --- a/seeker/snippet/polygon_table_from_datafile.py +++ /dev/null @@ -1,36 +0,0 @@ -#date: 2024-09-03T16:56:27Z -#url: https://api.github.com/gists/4e27bb8bae3bbff729866e1bc5866f8c -#owner: https://api.github.com/users/SeanLikesData - -# code snippets for generating polygon tables by reading various file types. -# Note that the only real change is the ST_GeomFrom{filetype} function in line 7. - -# ST_GeomfromText -polygon_wkt_df = sedona.read.format("csv").\ - option("delimiter", "\t").\ - option("header", "false").\ - load("data/county_small.tsv") - -polygon_wkt_df.createOrReplaceTempView("polygontable") -polygon_df = sedona.sql("select polygontable._c6 as name, ST_GeomFromText(polygontable._c0) as countyshape from polygontable") -polygon_df.show(5) - -# ST_GeomFromWKB -polygon_wkb_df = sedona.read.format("csv").\ - option("delimiter", "\t").\ - option("header", "false").\ - load("data/county_small_wkb.tsv") - -polygon_wkb_df.createOrReplaceTempView("polygontable") -polygon_df = sedona.sql("select polygontable._c6 as name, ST_GeomFromWKB(polygontable._c0) as countyshape from polygontable") -polygon_df.show(5) - -# ST_GeomFromGeoJSON -polygon_json_df = sedona.read.format("csv").\ - option("delimiter", "\t").\ - option("header", "false").\ - load("data/testPolygon.json") - -polygon_json_df.createOrReplaceTempView("polygontable") -polygon_df = sedona.sql("select ST_GeomFromGeoJSON(polygontable._c0) as countyshape from polygontable") -polygon_df.show(5) diff --git a/seeker/snippet/sedona_df_to_geopandas.py b/seeker/snippet/sedona_df_to_geopandas.py deleted file mode 100644 index 8aa65e64..00000000 --- a/seeker/snippet/sedona_df_to_geopandas.py +++ /dev/null @@ -1,7 +0,0 @@ -#date: 2024-09-03T16:56:27Z -#url: https://api.github.com/gists/4e27bb8bae3bbff729866e1bc5866f8c -#owner: https://api.github.com/users/SeanLikesData - -# Convert to geopandas -df = sedona_df.toPandas() -gdf = gpd.GeoDataFrame(df, geometry="geom") \ No newline at end of file diff --git a/seeker/snippet/snowpark_session.py b/seeker/snippet/snowpark_session.py new file mode 100644 index 00000000..64fb5d66 --- /dev/null +++ b/seeker/snippet/snowpark_session.py @@ -0,0 +1,28 @@ +#date: 2024-09-05T16:57:47Z +#url: https://api.github.com/gists/8818dafe04307567ce0df7c318a9bd04 +#owner: https://api.github.com/users/louspringer + +# This code snippet creates a Snowflake Snowpark session. It works equivalently to obtain a +# session in a Snowflake hosted notebook, python script or Streamlit application, without +# the ~/.snowsql/config configuration. +# +# It uses the 'connection_name' set to 'default' to configure and get or create the session. +# The session is initialized in a concise one-liner. +# +# Local Configuration: +# Ensure the following is set up in the ~/.snowsql/config file: +# +# [connections.default] +# accountname = +# username = +# private_key_path = # Optional if you're using keypair auth +# warehouse = +# database = +# schema = +# role = +# +# Replace with your actual Snowflake account details. +# The 'default' profile will pull these connection parameters automatically when creating the session. + +from snowflake.snowpark import Session +session = Session.builder.config(key="connection_name", value="default").getOrCreate() diff --git a/seeker/snippet/spatial_joins.py b/seeker/snippet/spatial_joins.py deleted file mode 100644 index 1b762a3f..00000000 --- a/seeker/snippet/spatial_joins.py +++ /dev/null @@ -1,60 +0,0 @@ -#date: 2024-09-03T16:56:27Z -#url: https://api.github.com/gists/4e27bb8bae3bbff729866e1bc5866f8c -#owner: https://api.github.com/users/SeanLikesData - -# Distance Join - -point_csv_df_1 = sedona.read.format("csv").\ - option("delimiter", ",").\ - option("header", "false").load("data/testpoint.csv") - -point_csv_df_1.createOrReplaceTempView("pointtable") - -point_df1 = sedona.sql("SELECT ST_Point(cast(pointtable._c0 as Decimal(24,20)),cast(pointtable._c1 as Decimal(24,20))) as pointshape1, \'abc\' as name1 from pointtable") -point_df1.createOrReplaceTempView("pointdf1") - -point_csv_df2 = sedona.read.format("csv").\ - option("delimiter", ",").\ - option("header", "false").load("data/testpoint.csv") - -point_csv_df2.createOrReplaceTempView("pointtable") -point_df2 = sedona.sql("select ST_Point(cast(pointtable._c0 as Decimal(24,20)),cast(pointtable._c1 as Decimal(24,20))) as pointshape2, \'def\' as name2 from pointtable") -point_df2.createOrReplaceTempView("pointdf2") - -distance_join_df = sedona.sql("select * from pointdf1, pointdf2 where ST_Distance(pointdf1.pointshape1,pointdf2.pointshape2) < 2") -distance_join_df.explain() -distance_join_df.show(5) - -# Range Join - -import pandas as pd -gdf = gpd.read_file("data/gis_osm_pois_free_1.shp") -gdf = gdf.replace(pd.NA, '') -osm_points = sedona.createDataFrame( - gdf -) -osm_points.createOrReplaceTempView("points") - -transformed_df = sedona.sql( - """ - SELECT osm_id, - code, - fclass, - name, - ST_Transform(geometry, 'epsg:4326', 'epsg:2180') as geom - FROM points - """) - -transformed_df.createOrReplaceTempView("points_2180") - -neighbours_within_1000m = sedona.sql(""" - SELECT a.osm_id AS id_1, - b.osm_id AS id_2, - a.geom - FROM points_2180 AS a, points_2180 AS b - WHERE ST_Distance(a.geom,b.geom) < 50 - """) - -# Convert to geopandas -df = neighbours_within_1000m.toPandas() -gdf = gpd.GeoDataFrame(df, geometry="geom") \ No newline at end of file diff --git a/seeker/snippet/tap-linux.py b/seeker/snippet/tap-linux.py new file mode 100644 index 00000000..3ac2357a --- /dev/null +++ b/seeker/snippet/tap-linux.py @@ -0,0 +1,59 @@ +#date: 2024-09-05T16:58:25Z +#url: https://api.github.com/gists/0930e7da776c6690bf744233065008eb +#owner: https://api.github.com/users/thesle3p + +# Simple linux tun/tap device example tunnel over udp +# create tap device with ip tuntap add device0 tap +# set ip address on it and run tap-linux on that device and set desitation ip +# run same on another node, changing dst ip to first node + +import fcntl +import struct +import os +import socket +import threading +import sys + +TUNSETIFF = 0x400454ca +TUNSETOWNER = TUNSETIFF + 2 +IFF_TUN = 0x0001 +IFF_TAP = 0x0002 +IFF_NO_PI = 0x1000 + + +def udp_send(dst, packet): + print "udp_send" + sock.sendto(packet, (dst, 40000)) + +def recv(): + ss = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + ss.bind(("0.0.0.0", 40000)) + while True: + data, addr = ss.recvfrom(1024) + print "udp_recv" + os.write(tun.fileno(), data) + +if __name__ == "__main__": + + if len(sys.argv) < 3: + print "Usage: tap-linux.py " + sys.exit(1) + iface = sys.argv[1] + dst = sys.argv[2] + print "Working on %s inteface, destination address %s:40000 udp" % (iface, dst) + tun = open('/dev/net/tun', 'r+b') + ifr = struct.pack('16sH', iface, IFF_TAP | IFF_NO_PI) + fcntl.ioctl(tun, TUNSETIFF, ifr) + fcntl.ioctl(tun, TUNSETOWNER, 1000) + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP + t = threading.Thread(target=recv) + try: + t.start() + while True: + packet = os.read(tun.fileno(), 2048) + if True: + udp_send(dst, packet) + + except KeyboardInterrupt: + print "Terminating ..." + os._exit(0) \ No newline at end of file