Skip to content

Commit

Permalink
make downloading work on Windows. Unzipping doesn't work though
Browse files Browse the repository at this point in the history
  • Loading branch information
HugoGranstrom committed May 11, 2021
1 parent ee45554 commit f37f0b0
Showing 1 changed file with 43 additions and 6 deletions.
49 changes: 43 additions & 6 deletions install/torch_installer.nim
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,11 @@
import
std/[asyncdispatch, httpclient,
strformat, strutils, os],
#zippy/ziparchives,
zip/zipfiles

{.passl: "-lz".}

This comment has been minimized.

Copy link
@Clonkk

Clonkk May 17, 2021

Member

Why not use Zippy ?
Unzip to a tmp folder and moveDir the un-zipped content back to vendor as a workaround of guzba/zippy#8

It should work on Windows.

This comment has been minimized.

Copy link
@HugoGranstrom

HugoGranstrom May 18, 2021

Author Member

The problem with Zippy is that it reads the entire file into memory at once. And being a 3GB compressed file it takes up at minimum 6GB (probably quite a bit more) of RAM when it is unzipping it. I wasn't able to unzip the file on my 8GB laptop without getting an Out of memory error. The ordinary Zip package on the other hand unzips the file in a stream instead if I understand it correctly and doesn't use that much RAM. So until we can use Zippy it needs to get some sort of Stream support.

This comment has been minimized.

Copy link
@Clonkk

Clonkk May 18, 2021

Member

Ah okay, good point then :)


type
Acceleration* = enum
Cpu = "cpu"
Expand All @@ -26,17 +29,17 @@ type
proc getProjectDir(): string {.compileTime.} =
currentSourcePath.rsplit(DirSep, 1)[0]

proc onProgressChanged(total, progress, speed: BiggestInt) {.async.} =
proc onProgressChanged(total, progress, speed: BiggestInt) =
echo &"Downloaded {progress} of {total}"
echo &"Current rate: {speed.float64 / (1000*1000):4.3f} MiBi/s" # TODO the unit is neither MB or Mb or MiBi ???

proc downloadTo(url, targetDir, filename: string) {.async.} =
var client = newAsyncHttpClient()
proc downloadTo(url, targetDir, filename: string) =
var client = newHttpClient()
defer: client.close()
client.onProgressChanged = onProgressChanged
echo "Starting download of \"", url, '\"'
echo "Storing temporary into: \"", targetDir, '\"'
await client.downloadFile(url, targetDir / filename)
client.downloadFile(url, targetDir / filename)

proc getUrlAndFilename(version = "1.8.1", accel = Cuda111, abi = Cpp11): tuple[url, filename: string] =
result.filename = "libtorch-"
Expand All @@ -47,16 +50,50 @@ proc getUrlAndFilename(version = "1.8.1", accel = Cuda111, abi = Cpp11): tuple[u
result.filename &= &"%2B{accel}"
result.filename &= ".zip"
elif defined(windows):
let abi = Cpp
doAssert abi == Cpp, "LibTorch for Windows does not support the C++11 ABI"
result.filename &= &"-win-{abi}-{version}.zip"
result.filename &= &"win-{abi}-{version}"
result.filename &= &"%2B{accel}"
result.filename &= ".zip"
elif defined(osx):
doAssert accel == Cpu, "LibTorch for MacOS does not support GPU acceleration"
result.filename &= &"macos-{version}.zip"

result.url = &"https://download.pytorch.org/libtorch/{accel}/{result.filename}"

proc downloadLibTorch(url, targetDir, filename: string) =
waitFor url.downloadTo(targetDir, filename)
if not fileExists(targetDir / "libtorch.zip"):

This comment has been minimized.

Copy link
@Clonkk

Clonkk May 17, 2021

Member

@HugoGranstrom Why is this hardcoded to "libtorch.zip" instead of using filename ?

This comment has been minimized.

Copy link
@HugoGranstrom

HugoGranstrom May 17, 2021

Author Member

It's left since I tried to debug why it didn't work, I'll fix it. Good catch! :)

This comment has been minimized.

Copy link
@HugoGranstrom

HugoGranstrom May 17, 2021

Author Member

Pushed a fix now

url.downloadTo(targetDir, "libtorch.zip")
else:
echo "File is already downloaded"

#[ proc uncompress(targetDir, filename: string, delete = false) =
let tmpZip = targetDir / filename
var tmpDir = getTempDir() / "libtorch_temp_download"
var i = 0
while dirExists(tmpDir & $i):
inc(i)
tmpDir = tmpDir & $i
echo "Tempdir: ", tmpDir
let folderName = filename[0 ..< ^4] # remove .zip
let targetDir = targetDir# / "test"
#removeDir(tmpDir)
#createDir(tmpDir)
echo "Decompressing \"", tmpZip, "\" and storing into \"", targetDir, "\""
extractAll(tmpZip, tmpDir)
echo "Extraction done! Now copying from temp"
removeDir(targetDir / "libtorch") # remove old install
echo "Removed old folder"
copyDir(tmpDir, targetDir)
echo "Copy done!"
removeDir(tmpDir)
echo "Done."
if delete:
echo "Deleting \"", tmpZip, "\""
removeFile(tmpZip)
else:
echo "Not deleting \"", tmpZip, "\"" ]#

proc uncompress(targetDir, filename: string, delete = false) =
var z: ZipArchive
Expand Down

0 comments on commit f37f0b0

Please sign in to comment.