From f162ae6b9b6a560935ff6d9f23d2c2d7ac69f123 Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:20:41 -0500 Subject: [PATCH 1/3] short term trend page --- _bookdown.yml | 1 + chapters/short_term_trend.Rmd | 50 +++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 chapters/short_term_trend.Rmd diff --git a/_bookdown.yml b/_bookdown.yml index d4ff6255..7d253e35 100644 --- a/_bookdown.yml +++ b/_bookdown.yml @@ -7,6 +7,7 @@ rmd_files: - "chapters/sectionHeaders/methods.rmd" - "chapters/erddap_query_and_build.Rmd" - "chapters/Trend_analysis.Rmd" + - "chapters/short_term_trend.Rmd" - "chapters/regime_shift_analysis.Rmd" - "chapters/survdat.rmd" - "chapters/EPU.Rmd" diff --git a/chapters/short_term_trend.Rmd b/chapters/short_term_trend.Rmd new file mode 100644 index 00000000..374ab17b --- /dev/null +++ b/chapters/short_term_trend.Rmd @@ -0,0 +1,50 @@ +# Short Term Trend Analysis + + +**Description**: Time series trend analysis for short time series + +**Found in**: State of the Ecosystem - Gulf of Maine & Georges Bank (2025+), State of the Ecosystem - Mid-Atlantic (2025+) + +**Indicator category**: Extensive analysis, not yet published + +**Contributor(s)**: Andy Beet + +**Data steward**: NA + +**Point of contact**: Andy Beet, + +**Public availability statement**: NA + + +## Methods + +In prep: **A.Beet "A test for short term trend detection in the presence of autocorrelation"** + +The specific model addressed here is of the form, + +\begin{equation} + Y_t = \beta_0 + \beta_1 t + \epsilon_t +\end{equation} + +where $\epsilon_t = \phi\epsilon_{t-1} + z_t$ is a stationary first order autoregressive process with $z_t \sim N(0,\sigma^2)$. Interest centers on testing the null hypothesis, $H_0:\beta_1 = 0$ against the alternative, $H_1:\beta_1 \neq 0$ + +Testing for a trend in time series data has been addressed by many authors from a wide range of disciplines including economics, statistics, hydrology, ecology, fisheries, and epidemiology; [@cochrane_application_1949; @prais_trend_1954; @beach_maximum_1978; @park_estimating_1980; @brillinger_trend_1994; @bence_analysis_1995; @woodward_improved_1997; @zhang_temperature_2000; @yue_influence_2002; @wang_linear_2015 ;@hardison_simulation_2019]. These approaches have typically taken one of three paths; non parametric methods such as the Mann Kendall test and its pre-whitening variants @hamed_modified_1998; @zhang_temperature_2000; @yue_applicability_2002; @wang_linear_2015); parametric methods involving data transformation such as @cochrane_application_1949, @prais_trend_1954, @woodward_improved_1997); parametric methods such as generalized least squares and maximum likelihood estimation (@beach_maximum_1978; @davison_economic_1999, @pinheiro_mixed_2000). + +Is has been well documented that under the null hypothesis of no trend, $H_0:\beta_1=0$, in the presence of autocorrelation, parametric tests relying on asymptotic distribution theory reject the null hypothesis too frequently, leading to nominal significance levels that are too high, even for relatively long time series of length n = 100 (@woodward_improved_1997). Non parametric tests like those listed above also suffer the same problem. + +We introduce a test that, like @beach_maximum_1978, uses maximum likelihood for parameter estimation, but differs in that the significance of the likelihood ratio statistic, LR, is assessed via a parametric bootstrap (@efron_introduction_1993). Parametric bootstrap procedures have been used in some of the aforementioned work. @woodward_improved_1997 uses an alternative statistic, the Cochrane-Orchutt statistic, for detecting a trend. @rayner_bootstrapping_1990 focuses on the significance of the AR(1) parameter and @bence_analysis_1995 focuses on adjusting confidence intervals. We use the parametric bootstrap as an alternative means of assessing the significance of the LR statistic. + +The Likelihood ratio statistic combined with a parametric bootstrap is employed to test for a linear trend in the presence of autocorrelation in the form of an AR(1) process. Small samples of size, n = 10, are of particular interest. + +### Data source(s) +NA + +### Data extraction +NA + +### Data analysis + +Code used for the fitting and evaluation of short term trend can be found [here](https://github.com/NOAA-EDAB/arfit). + +**catalog link** +No associated catalog page \ No newline at end of file From 2a9d1708e5bffe510b853e5a1b62228ff3cdcf57 Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:26:32 -0500 Subject: [PATCH 2/3] short term trend bibliography --- bibliography/short_term_trend.bib | 328 ++++++++++++++++++++++++++++++ index.Rmd | 2 +- packages.bib | 2 +- 3 files changed, 330 insertions(+), 2 deletions(-) create mode 100644 bibliography/short_term_trend.bib diff --git a/bibliography/short_term_trend.bib b/bibliography/short_term_trend.bib new file mode 100644 index 00000000..1826cf6b --- /dev/null +++ b/bibliography/short_term_trend.bib @@ -0,0 +1,328 @@ +@article{cochrane_application_1949, + title = {Application of {Least} {Squares} {Regression} to {Relationships} {Containing} {Auto}-{Correlated} {Error} {Terms}}, + volume = {44}, + issn = {0162-1459}, + url = {https://doi.org/10.1080/01621459.1949.10483290}, + doi = {10.1080/01621459.1949.10483290}, + abstract = {We point out that autocorrelated error terms require modification of the usual methods of estimation and prediction; and we present evidence showing that the error terms involved in most current formulations of economic relations are highly positively autocorrelated. In doing this we demonstrate that when estimates of autoregressive properties of error terms are based on calculated residuals there is a large bias towards randomness. We demonstrate how much efficiency may be lost by current methods of estimation and prediction; and we give a tentative method of procedure for regaining the lost efficiency. * We wish to express our thanks for the considerable assistance we have received from Richard Stone.}, + number = {245}, + urldate = {2021-07-21}, + journal = {Journal of the American Statistical Association}, + author = {Cochrane, D. and Orcutt, G. H.}, + month = mar, + year = {1949}, + note = {Publisher: Taylor \& Francis +\_eprint: https://doi.org/10.1080/01621459.1949.10483290}, + pages = {32--61}, +} + +@article{davison_economic_1999, + year = {1999}, + title = {Economic Theory and Methods.}, + author = {Davidson, R. and Mackinnon, J. G.}, + journal = {Oxford University Press, New York}, + +} + +@article{efron_introduction_1993, + year = {1993}, + title = {An Inrtoduction to the Bootstrap}, + author = {Efron, B. and Tibshirani R. J.}, + journal = {Chapman and Hall, New York}, + +} + +@article{silvey_statistical_1980, + year = {1980}, + title = {Statistical Inference}, + author = {Silvey, S. D.}, + journal = {Chapman and Hall, New York}, + +} + + + +@article{pinheiro_mixed_2000, + year = {2000}, + title = {Mixed-Effects Models in S and S-PLUS.}, + author = {Pinheiro, J.C. and Bates, D.M.}, + journal = {Springer, New York}, + +} + + +@article{prais_trend_1954, + title = {Trend Estimators and Serial Correlation.}, + volume = {No. 383. Chicago}, + year = {1954}, + journal = {Cowles Commission Discussion Paper}, + author = {Prais, S. J. and Winsten, C. B.}, +} + +@article{park_estimating_1980, + title = {Estimating the autocorrelated error model with trended data}, + volume = {13}, + issn = {0304-4076}, + url = {https://www.sciencedirect.com/science/article/pii/0304407680900147}, + doi = {10.1016/0304-4076(80)90014-7}, + abstract = {A Monte Carlo study of the small sample properties of various estimators of the linear regression model with first-order autocorrelated errors. When independent variables are trended, estimators using Ttransformed observations (Prais-Winsten) are much more efficient than those using T–1 (Cochrane–Orcutt). The best of the feasible estimators isiterated Prais-Winsten using a sum-of-squared-error minimizing estimate of the autocorrelation coefficient ϱ. None of the feasible estimators performs well in hypothesis testing; all seriously underestimate standard errors, making estimated coefficients appear to be much more significant than they actually are.}, + language = {en}, + number = {2}, + urldate = {2021-07-21}, + journal = {Journal of Econometrics}, + author = {Park, Rolla Edward and Mitchell, Bridger M.}, + month = jun, + year = {1980}, + pages = {185--201}, +} + +@article{zhang_temperature_2000, + title = {Temperature and precipitation trends in {Canada} during the 20th century}, + volume = {38}, + issn = {0705-5900}, + url = {https://doi.org/10.1080/07055900.2000.9649654}, + doi = {10.1080/07055900.2000.9649654}, + abstract = {Trends in Canadian temperature and precipitation during the 20th century are analyzed using recently updated and adjusted station data. Six elements, maximum, minimum and mean temperatures along with diurnal temperature range (DTR), precipitation totals and ratio of snowfall to total precipitation are investigated. Anomalies from the 1961–1990 reference period were first obtained at individual stations, and were then used to generate gridded datasets for subsequent trend analyses. Trends were computed for 1900–1998 for southern Canada (south of 60°N), and separately for 1950–1998 for the entire country, due to insufficient data in the high arctic prior to the 1950s. From 1900–1998, the annual mean temperature has increased between 0.5 and 1.5°C in the south. The warming is greater in minimum temperature than in maximum temperature in the first half of the century, resulting in a decrease of DTR. The greatest warming occurred in the west, with statistically significant increases mostly seen during spring and summer periods. Annual precipitation has also increased from 5\% to 35\% in southern Canada over the same period. In general, the ratio of snowfall to total precipitation has been increasing due mostly to the increase in winter precipitation which generally falls as snow and an increase of ratio in autumn. Negative trends were identified in some southern regions during spring. From 1950–1998, the pattern of temperature change is distinct: warming in the south and west and cooling in the northeast, with similar magnitudes in both maximum and minimum temperatures. This pattern is mostly evident in winter and spring. Across Canada, precipitation has increased by 5\% to 35\%, with significant negative trends found in southern regions during winter. Overall, the ratio of snowfall to total precipitation has increased, with significant negative trends occurring mostly in southern Canada during spring. Indices of abnormal climate conditions are also examined. These indices were defined as areas of Canada for 1950–1998, or southern Canada for 1900–1998, with temperature or precipitation anomalies above the 66th or below the 34th percentiles in their relevant time series. These confirmed the above findings and showed that climate has been becoming gradually wetter and warmer in southern Canada throughout the entire century, and in all of Canada during the latter half of the century.}, + number = {3}, + urldate = {2021-07-21}, + journal = {Atmosphere-Ocean}, + author = {Zhang, Xuebin and Vincent, Lucie A. and Hogg, W. D. and Niitsoo, Ain}, + month = sep, + year = {2000}, + note = {Publisher: Taylor \& Francis +\_eprint: https://doi.org/10.1080/07055900.2000.9649654}, + pages = {395--429}, +} + + +@article{hamed_modified_1998, + title = {A modified {Mann}-{Kendall} trend test for autocorrelated data}, + volume = {204}, + issn = {00221694}, + url = {https://linkinghub.elsevier.com/retrieve/pii/S002216949700125X}, + doi = {10.1016/S0022-1694(97)00125-X}, + abstract = {One of the commonly used tools for detecting changes in climatic and hydrologic time series is trend analysis. A number of statistical tests exist to assess the significance of trends in time series. One of the commonly used non-parametric trend tests is the Mann-Kendall trend test. The null hypothesis in the Mann-Kendall test is that the data are independent and randomly ordered. However, the existence of positive autocorrelation in the data increases the probability of detecting trends when actually none exist, and vice versa. Although this is a well-known fact, few studies have addressed this issue, and autocorrelation in the data is often ignored. In this study, the effect of autocorrelation on the variance of the Mann-Kendall trend test statistic is discussed. A theoretical relationship is derived to calculate the variance of the Mann-Kendall test statistic for autocorrelated data. The special cases of AR(I) and MA(1) dependence are discussed as examples. An approximation to the theoretical relationship is also presented in order to reduce computation time for long time series. Based on the modified value of the variance of the Mann Kendall trend test statistic, a modified non-parametric trend test which is suitable for autocorrelated data is proposed. The accuracy of the modified test in terms of its empirical significance level was found to be superior to that of the original Mann-Kendall trend test without any loss of power. The modified test is applied to rainfall as well as streamflow data to demonstrate its performance as compared to the original Mann-Kendalltrend test. © 1998 Elsevier Science B.V.}, + language = {en}, + number = {1-4}, + urldate = {2021-07-21}, + journal = {Journal of Hydrology}, + author = {Hamed, Khaled H. and Ramachandra Rao, A.}, + month = jan, + year = {1998}, + pages = {182--196}, +} + +@article{rayner_bootstrapping_1990, + title = {Bootstrapping p {Values} and {Power} in the {First}-{Order} {Autoregression}: {A} {Monte} {Carlo} {Investigation}}, + volume = {8}, + issn = {07350015}, + shorttitle = {Bootstrapping p {Values} and {Power} in the {First}-{Order} {Autoregression}}, + url = {https://www.jstor.org/stable/1391988?origin=crossref}, + doi = {10.2307/1391988}, + language = {en}, + number = {2}, + urldate = {2021-07-21}, + journal = {Journal of Business \& Economic Statistics}, + author = {Rayner, Robert K.}, + month = apr, + year = {1990}, + pages = {251}, + +} + + +@article{wang_linear_2015, + title = {Linear {Trend} {Detection} in {Serially} {Dependent} {Hydrometeorological} {Data} {Based} on a {Variance} {Correction} {Spearman} {Rho} {Method}}, + volume = {7}, + issn = {2073-4441}, + url = {http://www.mdpi.com/2073-4441/7/12/6673}, + doi = {10.3390/w7126673}, + abstract = {Hydrometeorological data are commonly serially dependent and thereby deviate from the assumption of independence that underlies the Spearman rho trend test. The presence of autocorrelation will influence the significance of observed trends. Specifically, the positive autocorrelation inflates Type I errors, while it deflates the power of trend detection in some cases. To address this issue, we derive a theoretical formula and recommend an appropriate empirical formula to calculate the rho variance of dependent series. The proposed procedure of the variance correction for the Spearman rho method is capable of mitigating the effect of autocorrelation on both, Type I error and power of the test. Similar to the Block Bootstrap method, it has the advantage that it does not require an initial knowledge of the autocorrelation structure or modification of the series. In comparison, the capability of the Pre-Whitening method is sensitive to model misspecification if the series are whitened by a parametric autocorrelation model. The Trend-Free Pre-Whitening method tends to lead to an overestimation of the statistical significance of trends, similar to the original Spearman rho test. The results of the study emphasize the importance of selecting a reliable method for trend detection in serially dependent data.}, + language = {en}, + number = {12}, + urldate = {2021-07-21}, + journal = {Water}, + author = {Wang, Wenpeng and Chen, Yuanfang and Becker, Stefan and Liu, Bo}, + month = dec, + year = {2015}, + pages = {7045--7065}, +} + +@article{woodward_improved_1997, + title = {Improved {Tests} for {Trend} in {Time} {Series} {Data}}, + volume = {2}, + issn = {10857117}, + url = {http://www.jstor.org/stable/1400511?origin=crossref}, + doi = {10.2307/1400511}, + language = {en}, + number = {4}, + urldate = {2021-07-21}, + journal = {Journal of Agricultural, Biological, and Environmental Statistics}, + author = {Woodward, Wayne A. and Bottone, Steven and Gray, H. L.}, + month = dec, + year = {1997}, + pages = {403}, + +} + +@article{yue_influence_2002, + title = {The influence of autocorrelation on the ability to detect trend in hydrological series}, + volume = {16}, + issn = {0885-6087, 1099-1085}, + url = {https://onlinelibrary.wiley.com/doi/10.1002/hyp.1095}, + doi = {10.1002/hyp.1095}, + abstract = {This study investigated using Monte Carlo simulation the interaction between a linear trend and a lag-one autoregressive (AR(1)) process when both exist in a time series. Simulation experiments demonstrated that the existence of serial correlation alters the variance of the estimate of the Mann–Kendall (MK) statistic; and the presence of a trend alters the estimate of the magnitude of serial correlation. Furthermore, it was shown that removal of a positive serial correlation component from time series by pre-whitening resulted in a reduction in the magnitude of the existing trend; and the removal of a trend component from a time series as a first step prior to pre-whitening eliminates the influence of the trend on the serial correlation and does not seriously affect the estimate of the true AR(1). These results indicate that the commonly used pre-whitening procedure for eliminating the effect of serial correlation on the MK test leads to potentially inaccurate assessments of the significance of a trend; and certain procedures will be more appropriate for eliminating the impact of serial correlation on the MK test. In essence, it was advocated that a trend first be removed in a series prior to ascertaining the magnitude of serial correlation. This alternative approach and the previously existing approaches were employed to assess the significance of a trend in serially correlated annual mean and annual minimum streamflow data of some pristine river basins in Ontario, Canada. Results indicate that, with the previously existing procedures, researchers and practitioners may have incorrectly identified the possibility of significant trends. Copyright  2002 Environment Canada. Published by John Wiley \& Sons, Ltd.}, + language = {en}, + number = {9}, + urldate = {2021-07-21}, + journal = {Hydrological Processes}, + author = {Yue, Sheng and Pilon, Paul and Phinney, Bob and Cavadias, George}, + month = jun, + year = {2002}, + pages = {1807--1829}, +} + + +@article{yue_applicability_2002, + title = {Applicability of prewhitening to eliminate the influence of serial correlation on the {Mann}-{Kendall} test: {TECHNICAL} {NOTE}}, + volume = {38}, + issn = {00431397}, + shorttitle = {Applicability of prewhitening to eliminate the influence of serial correlation on the {Mann}-{Kendall} test}, + url = {http://doi.wiley.com/10.1029/2001WR000861}, + doi = {10.1029/2001WR000861}, + language = {en}, + number = {6}, + urldate = {2021-07-21}, + journal = {Water Resources Research}, + author = {Yue, Sheng and Wang, Chun Yuan}, + month = jun, + year = {2002}, + pages = {4--1--4--7}, + +} + + + +@article{beach_maximum_1978, + title = {A {Maximum} {Likelihood} {Procedure} for {Regression} with {Autocorrelated} {Errors}}, + volume = {46}, + issn = {00129682}, + url = {https://www.jstor.org/stable/1913644?origin=crossref}, + doi = {10.2307/1913644}, + language = {en}, + number = {1}, + urldate = {2021-07-21}, + journal = {Econometrica}, + author = {Beach, Charles M. and MacKinnon, James G.}, + month = jan, + year = {1978}, + pages = {51}, +} + +@article{beach_full_1978, + title = {Full maximum likelihood estimation of second- order autoregressive error models}, + volume = {7}, + issn = {03044076}, + url = {https://linkinghub.elsevier.com/retrieve/pii/0304407678900684}, + doi = {10.1016/0304-4076(78)90068-4}, + language = {en}, + number = {2}, + urldate = {2021-07-21}, + journal = {Journal of Econometrics}, + author = {Beach, Charles M. and MacKinnon, James G.}, + month = jun, + year = {1978}, + pages = {187--198}, + file = {Beach and MacKinnon - 1978 - Full maximum likelihood estimation of second- orde.pdf:C\:\\Users\\andrew.beet\\Zotero\\storage\\LZF24Y9A\\Beach and MacKinnon - 1978 - Full maximum likelihood estimation of second- orde.pdf:application/pdf}, +} + + +@article{bence_analysis_1995, + title = {Analysis of {Short} {Time} {Series}: {Correcting} for {Autocorrelation}}, + volume = {76}, + issn = {00129658}, + shorttitle = {Analysis of {Short} {Time} {Series}}, + url = {http://doi.wiley.com/10.2307/1941218}, + doi = {10.2307/1941218}, + abstract = {Short time series are common in environmental and ecological studies. For sample sizes of 10 to 50, I examined the performance of methods for adjusting confidence intervals of the mean and parameters of a linear regression for autocorrelation. Similar analyses are common in econometric studies, and serious concerns have been raised about the adequacy of the common adjustment approaches, especially for estimating the slope of a linear regression when the explanatory variable has a time trend. Use of a bias-corrected estimate of the autocorrelation, either in an adjusted t test or in a two-stage approach, outperformed other methods, including maximum likelihood and bootstrap estimators, in terms of confidence interval coverage. The bias correction was, however, sometimes awkward to apply. It was generally better to test for autocorrelation at the 0.5 level and use ordinary least squares if the test was not significant, although this pretesting mainly helped for weak autocorrelation and small sample sizes. For the best methods, the coverage was sometimes still substantially less than the stated 95\% when autocorrelation was strong, even for sample sizes as large as 50. This was true for estimates of the mean, the regression intercept, and, when the explanatory variable had a time trend, the slope. Simulation results and an example show that different adjustment methods can produce substantially different estimates and confidence intervals. Cautious interpretation of confidence intervals and hypothesis tests is recommended.}, + language = {en}, + number = {2}, + urldate = {2021-07-21}, + journal = {Ecology}, + author = {Bence, James R.}, + month = mar, + year = {1995}, + pages = {628--639}, + +} + +@article{noguchi_bootstrap-based_2011, + title = {Bootstrap-based tests for trends in hydrological time series, with application to ice phenology data}, + volume = {410}, + issn = {00221694}, + url = {https://linkinghub.elsevier.com/retrieve/pii/S0022169411006299}, + doi = {10.1016/j.jhydrol.2011.09.008}, + abstract = {Studying trends in ice freeze-up and break-up dates as well as ice cover duration on lakes, which are sensitive to the air temperature, provides us useful information about climate change and linkages to atmospheric teleconnection patterns. Such data, however, typically show a positive serial correlation, which implies that a high/low observation tends to be followed by a high/low observation in the future. Such serial correlation among data is known to lead to deflation of p-values and related inflation of significance levels of most statistical procedures and tests, making the obtained results unreliable. In particular, we show that the classical statistical approaches, e.g., Student’s t- and Mann–Kendall tests, which assume independence of data, are inappropriate when a serial correlation is present. To overcome the problem, we suggest sieve bootstrap approaches which take into account the serial correlation of data for trend tracking in order to obtain more accurate and reliable estimates. We compare and discuss the results on the classical and newly proposed statistical approaches for the trend analysis using freeze-up and break-up dates as well as ice cover observations from Lake Baikal for the period 1869–1996 and Lake Kallavesi freeze-up dates data from 1834 to 1996. Remarkably, some recent studies of the Kallavesi freeze-up and the Baikal break-up dates, performed under the assumption of independence, report p-values that are relatively close to the border of statistical significance and, hence, provide some doubts in existence of a linear trend. In contrast, after taking into account the dependence structure of the observed data, the newly derived p-values have raised to an insignificant level, which implies no sufficient evidence for existence of a linear trend. The new bootstrap-based approaches also yield substantially less significant results for the existence of a trend for freeze-up dates and ice cover duration for Lake Baikal than reported earlier.}, + language = {en}, + number = {3-4}, + urldate = {2021-07-21}, + journal = {Journal of Hydrology}, + author = {Noguchi, Kimihiro and Gel, Yulia R. and Duguay, Claude R.}, + month = nov, + year = {2011}, + pages = {150--161}, + +} + +@article{brillinger_trend_1994, + title = {Trend analysis: {Time} series and point process problems}, + volume = {5}, + issn = {11804009, 1099095X}, + shorttitle = {Trend analysis}, + url = {https://onlinelibrary.wiley.com/doi/10.1002/env.3170050102}, + doi = {10.1002/env.3170050102}, + abstract = {The concern is with trend analysis. The data may be time series or point process. Parametric, semiparametric and non-parametric models and procedures are discussed. The problems and techniques are illustrated with examples taken from hydrology and seismology. There is review as well as some new analyses and proposals.}, + language = {en}, + number = {1}, + urldate = {2021-07-21}, + journal = {Environmetrics}, + author = {Brillinger, David R.}, + month = mar, + year = {1994}, + pages = {1--19}, + +} + + +@article{hardison_simulation_2019, + title = {A simulation study of trend detection methods for integrated ecosystem assessment}, + volume = {76}, + issn = {1054-3139, 1095-9289}, + url = {https://academic.oup.com/icesjms/article/76/7/2060/5512306}, + doi = {10.1093/icesjms/fsz097}, + abstract = {Abstract + The identification of trends in ecosystem indicators has become a core component of ecosystem approaches to resource management, although oftentimes assumptions of statistical models are not properly accounted for in the reporting process. To explore the limitations of trend analysis of short times series, we applied three common methods of trend detection, including a generalized least squares model selection approach, the Mann–Kendall test, and Mann–Kendall test with trend-free pre-whitening to simulated time series of varying trend and autocorrelation strengths. Our results suggest that the ability to detect trends in time series is hampered by the influence of autocorrelated residuals in short series lengths. While it is known that tests designed to account for autocorrelation will approach nominal rejection rates as series lengths increase, the results of this study indicate biased rejection rates in the presence of even weak autocorrelation for series lengths often encountered in indicators developed for ecosystem-level reporting (N = 10, 20, 30). This work has broad implications for ecosystem-level reporting, where indicator time series are often limited in length, maintain a variety of error structures, and are typically assessed using a single statistical method applied uniformly across all time series.}, + language = {en}, + number = {7}, + urldate = {2020-11-10}, + journal = {ICES Journal of Marine Science}, + author = {Hardison, Sean and Perretti, Charles T and DePiper, Geret S and Beet, Andrew}, + editor = {Coll, Marta}, + month = dec, + year = {2019}, + pages = {2060--2069}, + file = {Hardison et al. - 2019 - A simulation study of trend detection methods for .pdf:C\:\\Users\\andrew.beet\\Zotero\\storage\\6MHYMKEZ\\Hardison et al. - 2019 - A simulation study of trend detection methods for .pdf:application/pdf}, +} + +@book{hamilton_time_1994, + address = {Princeton, N.J}, + title = {Time series analysis}, + isbn = {978-0-691-04289-3}, + language = {en}, + publisher = {Princeton University Press}, + author = {Hamilton, James D.}, + year = {1994}, + keywords = {Time-series analysis}, + file = {Hamilton - 1994 - Time series analysis.pdf:C\:\\Users\\andrew.beet\\Zotero\\storage\\SEA945UN\\Hamilton - 1994 - Time series analysis.pdf:application/pdf}, +} + diff --git a/index.Rmd b/index.Rmd index 1be9e67f..c7a8a2bb 100644 --- a/index.Rmd +++ b/index.Rmd @@ -6,7 +6,7 @@ site: bookdown::bookdown_site knit: "bookdown::render_book" always_allow_html: true documentclass: book -bibliography: ["bibliography/introduction.bib","bibliography/aggregate_groups.bib","bibliography/seasonal_sst_anomaly_maps.bib","bibliography/Aquaculture.bib","bibliography/Bennet_indicator.bib","bibliography/bottom_temperature.bib","bibliography/bottom_temp_highres.bib","bibliography/Revenue_Diversity.bib","bibliography/ches_bay_water_quality.bib","bibliography/phytoplankton.bib","bibliography/ecosystem_overfishing.bib","bibliography/comm_eng.bib","bibliography/calanus_stage.bib","bibliography/ches_bay_temp.bib","bibliography/conceptmods.bib","bibliography/Condition.bib","bibliography/EPU.bib","bibliography/Expected_Number.bib","bibliography/cold_pool_index.bib","bibliography/sandlance.bib","bibliography/gulf_stream_index.bib","bibliography/habitat_diversity.bib","bibliography/habitat_vulnerability.bib","bibliography/Ich_div.bib","bibliography/long_term_sst.bib","bibliography/MAB_HAB.bib","bibliography/NE_HAB.bib","bibliography/habs.bib","bibliography/occupancy.bib","bibliography/productivity_tech_memo.bib","bibliography/RW.bib","bibliography/seabird_ne.bib","bibliography/seal_pup.bib","bibliography/slopewater_proportions.bib","bibliography/Species_dist.bib","bibliography/survey_data.bib","bibliography/thermal_hab_proj.bib","bibliography/trans_dates.bib","bibliography/trend_analysis.bib","bibliography/zooplankton.bib","bibliography/cold_pool_index.bib","bibliography/forage_energy_density.bib","bibliography/Forage_Fish_Biomass_Index.bib","bibliography/marine_heatwave.bib","bibliography/protected_species_hotspots.bib","bibliography/ocean_acidification.bib","bibliography/wind_habitat_occupancy.bib","bibliography/warm_core_rings.bib", "bibliography/glossary.bib","packages.bib"] +bibliography: ["bibliography/introduction.bib","bibliography/aggregate_groups.bib","bibliography/seasonal_sst_anomaly_maps.bib","bibliography/Aquaculture.bib","bibliography/Bennet_indicator.bib","bibliography/bottom_temperature.bib","bibliography/bottom_temp_highres.bib","bibliography/Revenue_Diversity.bib","bibliography/ches_bay_water_quality.bib","bibliography/phytoplankton.bib","bibliography/ecosystem_overfishing.bib","bibliography/comm_eng.bib","bibliography/calanus_stage.bib","bibliography/ches_bay_temp.bib","bibliography/conceptmods.bib","bibliography/Condition.bib","bibliography/EPU.bib","bibliography/Expected_Number.bib","bibliography/cold_pool_index.bib","bibliography/sandlance.bib","bibliography/gulf_stream_index.bib","bibliography/habitat_diversity.bib","bibliography/habitat_vulnerability.bib","bibliography/Ich_div.bib","bibliography/long_term_sst.bib","bibliography/MAB_HAB.bib","bibliography/NE_HAB.bib","bibliography/habs.bib","bibliography/occupancy.bib","bibliography/productivity_tech_memo.bib","bibliography/RW.bib","bibliography/seabird_ne.bib","bibliography/seal_pup.bib","bibliography/slopewater_proportions.bib","bibliography/Species_dist.bib","bibliography/survey_data.bib","bibliography/thermal_hab_proj.bib","bibliography/trans_dates.bib","bibliography/trend_analysis.bib","bibliography/zooplankton.bib","bibliography/cold_pool_index.bib","bibliography/forage_energy_density.bib","bibliography/Forage_Fish_Biomass_Index.bib","bibliography/marine_heatwave.bib","bibliography/protected_species_hotspots.bib","bibliography/ocean_acidification.bib","bibliography/wind_habitat_occupancy.bib","bibliography/warm_core_rings.bib","bibliography/short_term_trend.bib", "bibliography/glossary.bib","packages.bib"] geometry: "left=1.0in, right=1.0in, top=1.0in, bottom=1.0in, includefoot" biblio-style: apalike link-citations: true diff --git a/packages.bib b/packages.bib index ff0d63d4..8b2b614c 100644 --- a/packages.bib +++ b/packages.bib @@ -124,7 +124,7 @@ @Manual{R-servr @Manual{R-stocksmart, title = {stocksmart: Provides access to NOAAs stock SMART data}, author = {Andy Beet}, - note = {R package version 0.6.26}, + note = {R package version 0.6.29}, url = {https://github.com/NOAA-EDAB/stocksmart}, year = {2024}, } From 49a4467ddf6b403d86af8df0443760f06af5760b Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:46:39 -0500 Subject: [PATCH 3/3] updated reference formatting --- chapters/short_term_trend.Rmd | 4 ++-- packages.bib | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/chapters/short_term_trend.Rmd b/chapters/short_term_trend.Rmd index 374ab17b..1ce31a13 100644 --- a/chapters/short_term_trend.Rmd +++ b/chapters/short_term_trend.Rmd @@ -28,9 +28,9 @@ The specific model addressed here is of the form, where $\epsilon_t = \phi\epsilon_{t-1} + z_t$ is a stationary first order autoregressive process with $z_t \sim N(0,\sigma^2)$. Interest centers on testing the null hypothesis, $H_0:\beta_1 = 0$ against the alternative, $H_1:\beta_1 \neq 0$ -Testing for a trend in time series data has been addressed by many authors from a wide range of disciplines including economics, statistics, hydrology, ecology, fisheries, and epidemiology; [@cochrane_application_1949; @prais_trend_1954; @beach_maximum_1978; @park_estimating_1980; @brillinger_trend_1994; @bence_analysis_1995; @woodward_improved_1997; @zhang_temperature_2000; @yue_influence_2002; @wang_linear_2015 ;@hardison_simulation_2019]. These approaches have typically taken one of three paths; non parametric methods such as the Mann Kendall test and its pre-whitening variants @hamed_modified_1998; @zhang_temperature_2000; @yue_applicability_2002; @wang_linear_2015); parametric methods involving data transformation such as @cochrane_application_1949, @prais_trend_1954, @woodward_improved_1997); parametric methods such as generalized least squares and maximum likelihood estimation (@beach_maximum_1978; @davison_economic_1999, @pinheiro_mixed_2000). +Testing for a trend in time series data has been addressed by many authors from a wide range of disciplines including economics, statistics, hydrology, ecology, fisheries, and epidemiology (@cochrane_application_1949, @prais_trend_1954, @beach_maximum_1978, @park_estimating_1980, @brillinger_trend_1994, @bence_analysis_1995, @woodward_improved_1997, @zhang_temperature_2000, @yue_influence_2002, @wang_linear_2015 ,@hardison_simulation_2019). These approaches have typically taken one of three paths; non parametric methods such as the Mann Kendall test and its pre-whitening variants (@hamed_modified_1998, @zhang_temperature_2000, @yue_applicability_2002, @wang_linear_2015); parametric methods involving data transformation such as @cochrane_application_1949, @prais_trend_1954, @woodward_improved_1997; parametric methods such as generalized least squares and maximum likelihood estimation (@beach_maximum_1978, @davison_economic_1999, @pinheiro_mixed_2000). -Is has been well documented that under the null hypothesis of no trend, $H_0:\beta_1=0$, in the presence of autocorrelation, parametric tests relying on asymptotic distribution theory reject the null hypothesis too frequently, leading to nominal significance levels that are too high, even for relatively long time series of length n = 100 (@woodward_improved_1997). Non parametric tests like those listed above also suffer the same problem. +It has been well documented that under the null hypothesis of no trend, $H_0:\beta_1=0$, in the presence of autocorrelation, parametric tests relying on asymptotic distribution theory reject the null hypothesis too frequently, leading to nominal significance levels that are too high, even for relatively long time series of length n = 100 (@woodward_improved_1997). Non parametric tests like those listed above also suffer the same problem. We introduce a test that, like @beach_maximum_1978, uses maximum likelihood for parameter estimation, but differs in that the significance of the likelihood ratio statistic, LR, is assessed via a parametric bootstrap (@efron_introduction_1993). Parametric bootstrap procedures have been used in some of the aforementioned work. @woodward_improved_1997 uses an alternative statistic, the Cochrane-Orchutt statistic, for detecting a trend. @rayner_bootstrapping_1990 focuses on the significance of the AR(1) parameter and @bence_analysis_1995 focuses on adjusting confidence intervals. We use the parametric bootstrap as an alternative means of assessing the significance of the LR statistic. diff --git a/packages.bib b/packages.bib index 8b2b614c..f5e1b9ce 100644 --- a/packages.bib +++ b/packages.bib @@ -124,7 +124,7 @@ @Manual{R-servr @Manual{R-stocksmart, title = {stocksmart: Provides access to NOAAs stock SMART data}, author = {Andy Beet}, - note = {R package version 0.6.29}, + note = {R package version 0.6.30}, url = {https://github.com/NOAA-EDAB/stocksmart}, year = {2024}, }