Libraries (such as 'libtorch_cpu.so') and executables (such as 'torch_shm_manager')
get installed, quite surprisingly, to 'lib/python3.8/site-packages/{bin,lib}'.
Make sure RUNPATH matches that.

diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt
index 5b5622f0..30d27e57 100644
--- a/caffe2/CMakeLists.txt
+++ b/caffe2/CMakeLists.txt
@@ -1909,7 +1909,7 @@ if(BUILD_PYTHON)
   if(${BUILDING_WITH_TORCH_LIBS})
     # site-packages/caffe2/python/caffe2_pybind11_state
     # site-packages/torch/lib
-    set(caffe2_pybind11_rpath "${_rpath_portable_origin}/../../torch/lib")
+    set(caffe2_pybind11_rpath $ORIGIN/../../torch/lib)
   endif(${BUILDING_WITH_TORCH_LIBS})
 
   # Must also include `CMAKE_SHARED_LINKER_FLAGS` in linker flags for
diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake
index ee9cf410..f190e69b 100644
--- a/cmake/Dependencies.cmake
+++ b/cmake/Dependencies.cmake
@@ -4,7 +4,7 @@ if(APPLE)
   set(CMAKE_MACOSX_RPATH ON)
   set(_rpath_portable_origin "@loader_path")
 else()
-  set(_rpath_portable_origin $ORIGIN)
+  set(_rpath_portable_origin $ORIGIN/../lib)
 endif(APPLE)
 # Use separate rpaths during build and install phases
 set(CMAKE_SKIP_BUILD_RPATH  FALSE)
ree/nix/libutil?id=67de1fc87be8bb68c1d4e5211cb3e36a8acb9113'>tree</a><a href='/guix/commit/nix/libutil?id=67de1fc87be8bb68c1d4e5211cb3e36a8acb9113'>commit</a><a href='/guix/diff/nix/libutil?id=67de1fc87be8bb68c1d4e5211cb3e36a8acb9113'>diff</a></td><td class='form'><form class='right' method='get' action='/guix/log/nix/libutil'>
<input type='hidden' name='id' value='67de1fc87be8bb68c1d4e5211cb3e36a8acb9113'/><select name='qt'>
<option value='grep'>log msg</option>
<option value='author'>author</option>
<option value='committer'>committer</option>
<option value='range'>range</option>
</select>
<input class='txt' type='search' size='10' name='q' value=''/>
<input type='submit' value='search'/>
</form>
</td></tr></table>
<div class='path'>path: <a href='/guix/log/?id=67de1fc87be8bb68c1d4e5211cb3e36a8acb9113'>root</a>/<a href='/guix/log/nix?id=67de1fc87be8bb68c1d4e5211cb3e36a8acb9113'>nix</a>/<a href='/guix/log/nix/libutil?id=67de1fc87be8bb68c1d4e5211cb3e36a8acb9113'>libutil</a></div><div class='content'><table class='list nowrap'><tr class='nohover'><th class='left'>Age</th><th class='left'>Commit message (<a href='/guix/log/nix/libutil?id=67de1fc87be8bb68c1d4e5211cb3e36a8acb9113&amp;showmsg=1'>Expand</a>)</th><th class='left'>Author</th></tr>
<tr><td><span title='2020-12-08 22:30:07 +0100'>2020-12-08</span></td><td><a href='/guix/commit/nix/libutil?id=2e308238ad09e9527f9b8c3ec4a90b6b05a93367'>daemon: 'Agent' constructor takes a list of environment variables.</a><span class='msg-avail'>...</span></td><td>Ludovic Courtès</td></tr>
<tr><td><span title='2020-09-14 15:42:55 +0200'>2020-09-14</span></td><td><a href='/guix/commit/nix/libutil?id=ee9dff34f9317509cb2b833d07a0d5e01a36a4ae'>daemon: Move 'Agent' to libutil.</a><span class='msg-avail'>...</span></td><td>Ludovic Courtès</td></tr>
<tr><td><span title='2020-06-27 23:42:20 +0200'>2020-06-27</span></td><td><a href='/guix/commit/nix/libutil?id=8e6c1415d87272c0221ce328715fc0dd1dd3e032'>daemon: Recognize SHA3 and BLAKE2s.</a><span class='msg-avail'>...</span></td><td>Ludovic Courtès</td></tr>
<tr><td><span title='2020-06-27 23:42:20 +0200'>2020-06-27</span></td><td><a href='/guix/commit/nix/libutil?id=8dc6c387852bb9505be44567a5f56913633cc23a'>daemon: Remove OpenSSL hash compatibility wrappers.</a><span class='msg-avail'>...</span></td><td>Ludovic Courtès</td></tr>
<tr><td><span title='2020-06-27 23:42:20 +0200'>2020-06-27</span></td><td><a href='/guix/commit/nix/libutil?id=3fb6b8f30444d41963ba5bdd441123a6d2df17bd'>daemon: Map directly to gcrypt hash functions.</a><span class='msg-avail'>...</span></td><td>Ludovic Courtès</td></tr>
<tr><td><span title='2020-06-24 19:55:22 +0100'>2020-06-24</span></td><td><a href='/guix/commit/nix/libutil?id=cd43b55da7ab51c996a7a3449c78f2a493616106'>nix: Tweak .gitignore files.</a><span class='msg-avail'>...</span></td><td>Christopher Baines</td></tr>
<tr><td><span title='2020-03-26 12:59:33 +0100'>2020-03-26</span></td><td><a href='/guix/commit/nix/libutil?id=d0ed201e0a9736c64cb3c59d8c987c79fd6046d6'>daemon: Avoid kill -1 bug on the Hurd.</a><span class='msg-avail'>...</span></td><td>Manolis Ragkousis</td></tr>
<tr><td><span title='2020-01-12 00:19:09 +0100'>2020-01-12</span></td><td><a href='/guix/commit/nix/libutil?id=7033c7692ccbbbad8f7b9952015de071a5588e87'>daemon: Account for deleted store files when deduplication is on.</a><span class='msg-avail'>...</span></td><td>Ludovic Courtès</td></tr>
<tr><td><span title='2019-11-29 15:54:20 +0100'>2019-11-29</span></td><td><a href='/guix/commit/nix/libutil?id=b6b014bf42f68fcb71d37204328babb81e0e0855'>daemon: 'pathExists' uses 'statx' when available.</a><span class='msg-avail'>...</span></td><td>Ludovic Courtès</td></tr>
<tr><td><span title='2019-11-27 00:03:03 +0100'>2019-11-27</span></td><td><a href='/guix/commit/nix/libutil?id=fbecb5cddb2d36cc1a29dac3751b017fa531383c'>daemon: 'deletePath' uses 'statx' when available.</a><span class='msg-avail'>...</span></td><td>Ludovic Courtès</td></tr>
<tr><td><span title='2019-08-30 01:33:45 +0200'>2019-08-30</span></td><td><a href='/guix/commit/nix/libutil?id=546a709f202024c8a5173ad372a87ddc1c284c63'>daemon: Don't reply on 'st_blocks'.</a><span class='msg-avail'>...</span></td><td>Eelco Dolstra</td></tr>
<tr><td><span title='2018-12-16 16:28:07 +0100'>2018-12-16</span></td><td><a href='/guix/commit/nix/libutil?id=8327e733966cc4a93dfaadd1a761ca1088078baa'>daemon: Use unbranded phrases in comments and messages.</a><span class='msg-avail'>...</span></td><td>Ludovic Courtès</td></tr>
<tr><td><span title='2018-10-02 23:08:30 +0200'>2018-10-02</span></td><td><a href='/guix/commit/nix/libutil?id=7eca7892014d173ff523417138b90aeff5b7dec1'>daemon: Remove "case hack" for nars.</a><span class='msg-avail'>...</span></td><td>Ludovic Courtès</td></tr>
<tr><td><span title='2017-08-18 11:31:28 +0200'>2017-08-18</span></td><td><a href='/guix/commit/nix/libutil?id=ed464f48efd80e6439441d7a9f6d0e7988fb1272'>nix: Remove unused function.</a><span class='msg-avail'>...</span></td><td>Andy Wingo</td></tr>