evergreen/
remote.rs

1use crate as eg;
2use eg::Editor;
3use eg::EgResult;
4use ftp::FtpStream;
5use glob;
6use std::cmp;
7use std::env;
8use std::fmt;
9use std::fs;
10use std::io::Read;
11use std::io::Write;
12use std::net::TcpStream;
13use std::net::ToSocketAddrs;
14use std::path::Path;
15use std::path::PathBuf;
16use std::time::Duration;
17use url::Url;
18
19#[derive(Debug, Clone, Copy, PartialEq)]
20pub enum Proto {
21    Sftp,
22    Ftp,
23}
24
25impl From<Proto> for &'static str {
26    fn from(p: Proto) -> &'static str {
27        match p {
28            Proto::Sftp => "sftp",
29            Proto::Ftp => "ftp",
30        }
31    }
32}
33
34pub struct RemoteAccount {
35    host: String,
36    port: Option<u16>,
37    username: Option<String>,
38    password: Option<String>,
39    proto: Proto,
40
41    remote_path: Option<PathBuf>,
42
43    sftp_session: Option<ssh2::Sftp>,
44
45    ftp_session: Option<ftp::FtpStream>,
46
47    /// Connect/read timeout
48    timeout: u32,
49
50    remote_account_id: Option<i64>,
51
52    /// Full path to an SSH private key file.
53    ssh_private_key: Option<PathBuf>,
54    ssh_private_key_password: Option<String>,
55
56    try_typical_ssh_keys: bool,
57}
58
59impl cmp::PartialEq for RemoteAccount {
60    /// Two accounts are the same if their connection details match.
61    fn eq(&self, other: &RemoteAccount) -> bool {
62        self.proto == other.proto
63            && self.host == other.host
64            && self.port == other.port
65            && self.username == other.username
66            && self.password == other.password
67            && self.remote_path == other.remote_path
68    }
69}
70
71impl fmt::Display for RemoteAccount {
72    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
73        write!(
74            f,
75            "RemoteAccount host={} user={}",
76            self.host,
77            self.username.as_deref().unwrap_or("")
78        )
79    }
80}
81
82impl RemoteAccount {
83    pub fn new(host: &str) -> RemoteAccount {
84        RemoteAccount {
85            host: host.to_string(),
86            port: None,
87            username: None,
88            password: None,
89            proto: Proto::Sftp,
90            remote_path: None,
91            timeout: 0,
92            ssh_private_key: None,
93            ssh_private_key_password: None,
94            sftp_session: None,
95            ftp_session: None,
96            try_typical_ssh_keys: true,
97            remote_account_id: None,
98        }
99    }
100
101    /// If read_mode is true, our remote_path = edi account "in_dir",
102    /// otherwise our remote_path = edi account "path"
103    pub fn from_edi_account(
104        editor: &mut Editor,
105        account_id: i64,
106        read_mode: bool,
107    ) -> EgResult<RemoteAccount> {
108        let edi_account = editor
109            .retrieve("acqedi", account_id)?
110            .ok_or_else(|| editor.die_event())?;
111
112        let mut account = RemoteAccount::from_url_string(edi_account["host"].str()?)?;
113
114        account.remote_account_id = Some(account_id);
115
116        account.remote_path = if read_mode {
117            edi_account["in_dir"].as_str().map(PathBuf::from)
118        } else {
119            edi_account["path"].as_str().map(PathBuf::from)
120        };
121
122        if let Some(username) = edi_account["username"].as_str() {
123            account.set_username(username);
124        }
125
126        if let Some(password) = edi_account["password"].as_str() {
127            account.set_password(password);
128        }
129
130        Ok(account)
131    }
132
133    /// Extract whatever components we can from a URL.
134    ///
135    /// Example "sftp://localhost"
136    pub fn from_url_string(url_str: &str) -> EgResult<RemoteAccount> {
137        let url = Url::parse(url_str).map_err(|e| format!("Cannot parse URL: {url_str} : {e}"))?;
138
139        let hostname = url.host_str().ok_or("URL has no host")?;
140        let mut account = RemoteAccount::new(hostname);
141
142        account.proto = match url.scheme() {
143            "sftp" => Proto::Sftp,
144            "ftp" | "" => Proto::Ftp,
145            _ => return Err(format!("Unsupported protocol: '{url_str}'").into()),
146        };
147
148        if !url.username().is_empty() {
149            account.set_username(url.username());
150        }
151
152        if !url.path().is_empty() {
153            account.remote_path = Some(PathBuf::from(url.path()));
154        }
155
156        Ok(account)
157    }
158
159    pub fn host(&self) -> &str {
160        &self.host
161    }
162
163    pub fn port(&self) -> Option<u16> {
164        self.port
165    }
166
167    pub fn proto(&self) -> Proto {
168        self.proto
169    }
170
171    pub fn username(&self) -> Option<&str> {
172        self.username.as_deref()
173    }
174
175    pub fn password(&self) -> Option<&str> {
176        self.password.as_deref()
177    }
178
179    pub fn remote_account_id(&self) -> Option<i64> {
180        self.remote_account_id
181    }
182
183    pub fn remote_path(&self) -> Option<&Path> {
184        self.remote_path.as_deref()
185    }
186
187    pub fn set_remote_path(&mut self, remote_path: &Path) {
188        self.remote_path = Some(remote_path.to_path_buf());
189    }
190
191    pub fn set_username(&mut self, username: &str) {
192        self.username = Some(username.to_string());
193    }
194
195    pub fn set_password(&mut self, password: &str) {
196        self.password = Some(password.to_string());
197    }
198
199    pub fn set_ssh_private_key(&mut self, keyfile: &Path) {
200        self.ssh_private_key = Some(keyfile.to_path_buf());
201    }
202
203    pub fn set_ssh_private_key_password(&mut self, keypass: &str) {
204        self.ssh_private_key_password = Some(keypass.to_string());
205    }
206
207    /// Set the global timeout for tasks that may block.
208    pub fn set_timeout(&mut self, timeout: u32) {
209        self.timeout = timeout;
210    }
211
212    /// Connect and authenticate with the remote site.
213    pub fn connect(&mut self) -> EgResult<()> {
214        log::debug!("{self} connect()");
215
216        match self.proto {
217            Proto::Sftp => self.connect_sftp(),
218            Proto::Ftp => self.connect_ftp(),
219        }
220    }
221
222    /// Returns a list of file paths matching our remote path and optional glob.
223    pub fn ls(&mut self) -> EgResult<Vec<PathBuf>> {
224        log::debug!(
225            "{self} ls() {:?}",
226            self.remote_path.as_ref().map(|p| p.display())
227        );
228
229        self.check_connected()?;
230
231        match self.proto {
232            Proto::Sftp => self.ls_sftp(),
233            Proto::Ftp => self.ls_ftp(),
234        }
235    }
236
237    /// Fetch a remote file by name, store the contents in a local
238    /// file, and return the created File handle.
239    pub fn get(&mut self, remote_path: &Path, local_path: &Path) -> EgResult<fs::File> {
240        log::debug!(
241            "{self} get() {} => {}",
242            remote_path.display(),
243            local_path.display()
244        );
245
246        self.check_connected()?;
247
248        match self.proto {
249            Proto::Sftp => self.get_sftp(remote_path, local_path),
250            Proto::Ftp => self.get_ftp(remote_path, local_path),
251        }
252    }
253
254    /// Returns an Err if we were never connected.
255    ///
256    /// This does not verify the connection is still open.
257    pub fn check_connected(&self) -> EgResult<()> {
258        match self.proto {
259            Proto::Sftp => {
260                if self.sftp_session.is_none() {
261                    return Err(format!("{self} is not connected to SFTP").into());
262                }
263            }
264            Proto::Ftp => {
265                if self.ftp_session.is_none() {
266                    return Err(format!("{self} is not connected to FTP").into());
267                }
268            }
269        }
270
271        Ok(())
272    }
273
274    /// Fetch a remote file by name, store the contents in a local
275    /// file, and return the created File handle.
276    fn get_sftp(&self, remote_path: &Path, local_path: &Path) -> EgResult<fs::File> {
277        let remote_filename = remote_path.display();
278
279        let mut remote_file = self
280            .sftp_session
281            .as_ref()
282            .unwrap()
283            .open(remote_path)
284            .map_err(|e| format!("Cannot open remote path {remote_filename} {e}"))?;
285
286        let mut bytes: Vec<u8> = Vec::new();
287        remote_file
288            .read_to_end(&mut bytes)
289            .map_err(|e| format!("Cannot read remote file: {remote_filename} {e}"))?;
290
291        self.write_local_file(local_path, &bytes)
292    }
293
294    /// Fetch a remote file by name, store the contents in a local
295    /// file, and return the created File handle.
296    fn get_ftp(&mut self, remote_path: &Path, local_path: &Path) -> EgResult<fs::File> {
297        let remote_filename = remote_path.display().to_string();
298
299        let cursor = self
300            .ftp_session
301            .as_mut()
302            .unwrap()
303            .simple_retr(&remote_filename)
304            .map_err(|e| format!("Cannot open remote file {remote_filename} {e}"))?;
305
306        let bytes = cursor.into_inner();
307
308        self.write_local_file(local_path, &bytes)
309    }
310
311    fn write_local_file(&self, local_path: &Path, bytes: &[u8]) -> EgResult<fs::File> {
312        let local_filename = local_path.display();
313
314        let mut local_file = fs::File::create(local_path)
315            .map_err(|e| format!("Cannot create local file {local_filename} {e}"))?;
316
317        local_file
318            .write_all(bytes)
319            .map_err(|e| format!("Cannot write to local file: {local_filename} {e}"))?;
320
321        Ok(local_file)
322    }
323
324    /// Returns a list of files/directories within our remote_path directory.
325    ///
326    /// If our remote_path contains a file name glob, the list only
327    /// includes files that match the glob.
328    fn ls_sftp(&self) -> EgResult<Vec<PathBuf>> {
329        let (remote_path, maybe_glob) = self.remote_path_and_glob()?;
330
331        log::info!("{self} listing directory {}", remote_path.display());
332
333        let contents = self
334            .sftp_session
335            .as_ref()
336            .unwrap()
337            .readdir(&remote_path)
338            .map_err(|e| {
339                format!(
340                    "{self} cannot list directory {} : {e}",
341                    remote_path.display()
342                )
343            })?;
344
345        let mut paths = Vec::new();
346
347        for (path, _) in contents {
348            if let Some(pattern) = maybe_glob.as_ref() {
349                if let Some(Some(name)) = path.file_name().map(|n| n.to_str()) {
350                    if pattern.matches(name) {
351                        paths.push(path);
352                    }
353                } else {
354                    log::warn!("{self} skipping non-stringifiable path: {path:?}");
355                }
356            } else {
357                paths.push(path);
358            }
359        }
360
361        Ok(paths)
362    }
363
364    /// Returns a list of files/directories within our remote_path directory.
365    ///
366    /// If our remote_path contains a file name glob, the list only
367    /// includes files that match the glob.
368    fn ls_ftp(&mut self) -> EgResult<Vec<PathBuf>> {
369        let (mut remote_path, maybe_glob) = self.remote_path_and_glob()?;
370
371        let remote_filename = remote_path.display().to_string();
372
373        log::info!("{self} listing directory {remote_filename}");
374
375        let contents = self
376            .ftp_session
377            .as_mut()
378            .unwrap()
379            .nlst(Some(&remote_filename))
380            .map_err(|e| format!("{self} cannot list directory {remote_filename} : {e}"))?;
381
382        let mut paths = Vec::new();
383
384        // nlist() returns the file name only, no path information.
385        // Reconstruct the path so we can return the fully qualified
386        // file name to the caller.
387
388        for file_name in contents {
389            remote_path.push(&file_name);
390
391            if let Some(pattern) = maybe_glob.as_ref() {
392                if pattern.matches(&file_name) {
393                    paths.push(remote_path.clone());
394                }
395            } else {
396                paths.push(remote_path.clone());
397            }
398
399            remote_path.pop(); // remove filename
400        }
401
402        Ok(paths)
403    }
404
405    fn connect_sftp(&mut self) -> EgResult<()> {
406        let port = self.port.unwrap_or(22);
407        let host = self.host.as_str();
408
409        let username = self
410            .username
411            .as_deref()
412            .ok_or("SFTP connection requires a username")?;
413
414        let tcp_result = if self.timeout > 0 {
415            let sock_addr = format!("{host}:{port}")
416                .to_socket_addrs()
417                .map_err(|e| format!("Cannot resolve host: {host} : {e}"))?
418                .next()
419                .ok_or_else(|| format!("Cannot resolve host: {host}"))?;
420
421            TcpStream::connect_timeout(&sock_addr, Duration::from_secs(self.timeout.into()))
422        } else {
423            TcpStream::connect((host, port))
424        };
425
426        let tcp_stream = tcp_result.map_err(|e| format!("Cannot connect to {host} : {e}"))?;
427
428        let mut sess = ssh2::Session::new()
429            .map_err(|e| format!("Cannot create SFTP session to {host} : {e}"))?;
430
431        if self.timeout > 0 {
432            sess.set_timeout(self.timeout * 1000); // ms
433        }
434
435        sess.set_tcp_stream(tcp_stream);
436
437        sess.handshake()
438            .map_err(|e| format!("SFTP handshake failed to {host} : {e}"))?;
439
440        if let Some(password) = self.password.as_ref() {
441            sess.userauth_password(username, password)
442                .map_err(|e| format!("Password auth failed for host {host}: {e}"))?;
443        } else {
444            self.sftp_key_auth(&mut sess, username)?;
445        }
446
447        if !sess.authenticated() {
448            return Err(format!("SFTP connection failed to authenticate with {host}").into());
449        }
450
451        let sftp = sess
452            .sftp()
453            .map_err(|e| format!("Cannot upgrade to SFTP connection for host {host}: {e}"))?;
454
455        self.sftp_session = Some(sftp);
456
457        log::info!("Successfully connected to SFTP at {host}");
458
459        Ok(())
460    }
461
462    fn connect_ftp(&mut self) -> EgResult<()> {
463        let port = self.port.unwrap_or(21);
464        let host = self.host.as_str();
465
466        let username = self
467            .username
468            .as_deref()
469            .ok_or("FTP connection requires a username")?;
470
471        let password = self
472            .password
473            .as_deref()
474            .ok_or("FTP connection requires a password")?;
475
476        let mut stream = FtpStream::connect(format!("{host}:{port}"))
477            .map_err(|e| format!("Cannot connect to host: {host}:{port} : {e}"))?;
478
479        if self.timeout > 0 {
480            stream
481                .get_ref()
482                .set_read_timeout(Some(Duration::from_secs(self.timeout.into())))
483                .map_err(|e| format!("Cannot set read timeout {} : {e}", self.timeout))?;
484        }
485
486        stream
487            .login(username, password)
488            .map_err(|e| format!("Login failed at host {host} for {username} : {e}"))?;
489
490        self.ftp_session = Some(stream);
491
492        Ok(())
493    }
494
495    /// Authenticate via ssh key file, trying the file provided and/or
496    /// typical local SSH key files.
497    fn sftp_key_auth(&self, sess: &mut ssh2::Session, username: &str) -> EgResult<()> {
498        let mut key_files = Vec::new();
499
500        if let Some(keyfile) = self.ssh_private_key.as_ref() {
501            key_files.push(keyfile.display().to_string());
502        }
503
504        if self.try_typical_ssh_keys {
505            if let Some(home) = env::vars().find(|(k, _)| k == "HOME").map(|(_, v)| v) {
506                let mut path_buf = PathBuf::new();
507
508                path_buf.push(home);
509                path_buf.push(".ssh");
510                path_buf.push("id_rsa");
511
512                key_files.push(path_buf.as_os_str().to_string_lossy().to_string());
513
514                path_buf.pop();
515                path_buf.push("dsa_rsa");
516
517                key_files.push(path_buf.as_os_str().to_string_lossy().to_string());
518            }
519        }
520
521        for key_file in key_files {
522            log::debug!("Trying key file {key_file}");
523
524            let result = sess.userauth_pubkey_file(
525                username,
526                None,
527                Path::new(&key_file),
528                self.ssh_private_key_password.as_deref(),
529            );
530
531            if result.is_ok() {
532                return Ok(());
533            }
534        }
535
536        Err("No suitable SSH keys found".into())
537    }
538
539    /// Return a directory path and a glob pattern if the provided path
540    /// contains a glob file name (e.g. /foo/bar/*.edi).   Otherwise,
541    /// returns None, meaning the originally provided path is the
542    /// one that should be used for send/recv files.
543    fn remote_path_and_glob(&self) -> EgResult<(PathBuf, Option<glob::Pattern>)> {
544        let remote_path = self.remote_path.as_deref().unwrap_or(Path::new("/"));
545
546        // Is there a trailing file name or is it just a directory?
547        let filename = match remote_path.file_name().map(|f| f.to_str()) {
548            Some(Some(f)) => f,
549            _ => return Ok((remote_path.to_path_buf(), None)),
550        };
551
552        // Does the file name contain a glob star
553        if !filename.contains('*') {
554            return Ok((remote_path.to_path_buf(), None));
555        }
556
557        // It's a glob.
558
559        let glob_pattern = glob::Pattern::new(filename)
560            .map_err(|e| format!("Invalid glob pattern: {filename} : {e}"))?;
561
562        let mut path_buf = PathBuf::new();
563
564        // Rebuild the path from its components then trim the globbed filename
565        for part in remote_path.iter() {
566            path_buf.push(part);
567        }
568
569        // Remove the filename part
570        path_buf.pop();
571
572        Ok((path_buf, Some(glob_pattern)))
573    }
574}
evergreen/remote.rs

evergreen/
remote.rs
